From 9a8a5cdfad3765f28665299b71d2eb6b364a43a6 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:43:29 -0500 Subject: [PATCH 001/117] docs(project): fix create URL flag example (#42) --- DOCUMENTATION.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 7e50c36..a0fa9a4 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -319,10 +319,10 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--target-url` against local addresses for fast feedback. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. ```bash -testsprite project create --name "Checkout" --target-url https://staging.example.com +testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` From 7928b5ce2f026f397c107bf4172469a1bba2f4b0 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:43:33 -0500 Subject: [PATCH 002/117] fix(help): include request-timeout in global flag guidance (#41) --- src/lib/output.ts | 4 +-- src/lib/render-error.test.ts | 12 ++++++++ src/lib/render-error.ts | 4 ++- test/__snapshots__/help.snapshot.test.ts.snap | 30 +++++++++---------- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/lib/output.ts b/src/lib/output.ts index 2b2db39..79b3f11 100644 --- a/src/lib/output.ts +++ b/src/lib/output.ts @@ -3,10 +3,10 @@ export type OutputMode = 'json' | 'text'; /** * Help-text footer pointing at the global options surface so users * looking at any subcommand `--help` don't miss `--dry-run`, `--output`, - * `--profile`, `--endpoint-url`, `--debug`. + * `--profile`, `--endpoint-url`, `--request-timeout`, `--debug`. */ export const GLOBAL_OPTS_HINT = - '\nGlobal options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug):' + + '\nGlobal options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug):' + '\n testsprite --help'; export function isOutputMode(value: unknown): value is OutputMode { diff --git a/src/lib/render-error.test.ts b/src/lib/render-error.test.ts index 2b1ced7..9635f2b 100644 --- a/src/lib/render-error.test.ts +++ b/src/lib/render-error.test.ts @@ -37,6 +37,12 @@ describe('rephraseUnknownOption', () => { expect(result).toContain('--endpoint-url'); }); + it('rephrases --request-timeout placed after subcommand', () => { + const result = rephraseUnknownOption("error: unknown option '--request-timeout'"); + expect(result).not.toBeNull(); + expect(result).toContain('--request-timeout'); + }); + it('rephrases --debug placed after subcommand', () => { const result = rephraseUnknownOption("error: unknown option '--debug'"); expect(result).not.toBeNull(); @@ -87,6 +93,12 @@ describe('rephraseUnknownOption', () => { expect(result).not.toBeNull(); expect(result).toContain('testsprite --endpoint-url '); }); + + it('value flag (--request-timeout) example DOES include a placeholder', () => { + const result = rephraseUnknownOption("error: unknown option '--request-timeout'"); + expect(result).not.toBeNull(); + expect(result).toContain('testsprite --request-timeout '); + }); }); describe('renderCommanderError', () => { diff --git a/src/lib/render-error.ts b/src/lib/render-error.ts index ebb4208..1898acd 100644 --- a/src/lib/render-error.ts +++ b/src/lib/render-error.ts @@ -11,13 +11,15 @@ import type { OutputMode } from './output.js'; * * `boolean` flags (--dry-run, --debug, --verbose) take no value; emit * example without a placeholder. `value` flags (--output, --profile, - * --endpoint-url) take a single argument; emit example with ``. + * --endpoint-url, --request-timeout) take a single argument; emit example + * with ``. */ const GLOBAL_FLAG_ARITY: Record = { 'dry-run': 'boolean', output: 'value', profile: 'value', 'endpoint-url': 'value', + 'request-timeout': 'value', debug: 'boolean', verbose: 'boolean', }; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6f1f158..6fee8af 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -36,7 +36,7 @@ Options: destroyed. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -49,7 +49,7 @@ List supported agent targets and skills, their status, and landing paths Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -86,7 +86,7 @@ exports[`--help snapshots > auth logout 1`] = ` Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -97,7 +97,7 @@ exports[`--help snapshots > auth whoami 1`] = ` Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -153,7 +153,7 @@ Get a project by id Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -175,7 +175,7 @@ Options: --max-items stop after this many items across auto-paged pages -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -280,7 +280,7 @@ Options: source body; json mode: wire envelope) -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -315,7 +315,7 @@ Options: per invocation; pin one yourself for safe retries. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -331,7 +331,7 @@ Options: --failed-only Keep only the failed step plus its immediate neighbors (±1) -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -344,7 +344,7 @@ Get a test by id Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -369,7 +369,7 @@ Options: --max-items stop after this many items across auto-paged pages -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -433,7 +433,7 @@ Dry-run shape notes: omit \`closure\` (or return it as null) since there is no dependency expansion. • \`autoHeal\` defaults true for FE reruns; BE reruns ignore the field entirely. -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -465,7 +465,7 @@ Options: --cursor with --history: opaque cursor from a prior page -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -516,7 +516,7 @@ Dependency-aware fresh run (M4): BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -539,7 +539,7 @@ Options: excluded when this flag is set. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; From 5f7dba1ca26d1bd3b45d1a6eaeabc0f5c4622e80 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:43:37 -0500 Subject: [PATCH 003/117] fix(auth): honor request timeout during configure (#52) --- src/commands/auth.test.ts | 44 +++++++++++++++++++++++++++++++++++++++ src/commands/auth.ts | 1 + 2 files changed, 45 insertions(+) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index d65e6cc..b360d2a 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -110,6 +110,50 @@ describe('runConfigure', () => { ); }); + it('uses requestTimeoutMs for the pre-write key validation ping', async () => { + const { deps } = makeCapture(); + let sawAbort = false; + const fetchImpl = vi.fn( + async (_input: string | URL | Request, init?: RequestInit) => + new Promise((_resolve, reject) => { + const signal = init?.signal; + const timeout = setTimeout(() => { + reject(new Error('requestTimeoutMs was not applied to the validation ping')); + }, 50); + signal?.addEventListener( + 'abort', + () => { + sawAbort = true; + clearTimeout(timeout); + reject(new DOMException('The operation timed out.', 'TimeoutError')); + }, + { once: true }, + ); + }), + ) as unknown as typeof fetch; + + await expect( + runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + requestTimeoutMs: 1, + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl, + }, + ), + ).rejects.toBeInstanceOf(CLIError); + + expect(sawAbort).toBe(true); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + it('throws VALIDATION_ERROR when --from-env is set but key is missing', async () => { const { deps } = makeCapture(); await expect( diff --git a/src/commands/auth.ts b/src/commands/auth.ts index de1eda4..18cd622 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -145,6 +145,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): baseUrl: facadeBaseUrl(apiUrl), apiKey, fetchImpl: deps.fetchImpl, + requestTimeoutMs: opts.requestTimeoutMs, }); try { // Tag the validation call with the originating command (when provided) so From 547b5be45799f5c8287638eb0ebc6a8bdef4cecb Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:43:42 -0500 Subject: [PATCH 004/117] fix(history): reject fractional page sizes (#55) --- src/commands/test.result.history.spec.ts | 26 ++++++++++++++++++++++++ src/commands/test.ts | 5 ++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index e3fb963..6f6cdd2 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -535,6 +535,32 @@ describe('runResultHistory — pagination', () => { expect(capturedUrl).toContain('pageSize=5'); }); + + it('rejects fractional --page-size before making a request', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => { + throw new Error('should not be called'); + }); + + await expect( + runResultHistory( + { + output: 'json', + testId: 'test_abc', + pageSize: 1.5, + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'page-size' }), + }); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/test.ts b/src/commands/test.ts index 08e4fe2..c61e84b 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3836,7 +3836,10 @@ export async function runResultHistory( // configured (codex round-2), matching validatePaginationFlags ordering // in `test list` / `project list`. if (opts.pageSize !== undefined) { - if (!Number.isFinite(opts.pageSize) || opts.pageSize < 1 || opts.pageSize > 100) { + if (!Number.isFinite(opts.pageSize) || !Number.isInteger(opts.pageSize)) { + throw localValidationError('page-size', 'must be an integer between 1 and 100'); + } + if (opts.pageSize < 1 || opts.pageSize > 100) { throw localValidationError('page-size', 'must be between 1 and 100'); } } From 91024301832cbba0e4beb2ef34ad72a8d85712cd Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:43:54 +0400 Subject: [PATCH 005/117] fix(target-url): treat trailing-dot hostnames as loopback in SSRF guard (#37) assertNotLocal lowercased the hostname but did not strip a trailing dot, so http://localhost. (the FQDN form of localhost, RFC 6761) and http://localhost%2e bypassed the host === 'localhost' loopback check. IP literals are already dot-normalized by the WHATWG URL parser, so only named hosts were affected. Strips one trailing dot before the comparison. Adds 4 regression tests (3 blocked variants + 1 public-FQDN no-false-positive). --- src/lib/target-url.spec.ts | 25 +++++++++++++++++++++++++ src/lib/target-url.ts | 9 ++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/lib/target-url.spec.ts b/src/lib/target-url.spec.ts index 96f705b..4998bc3 100644 --- a/src/lib/target-url.spec.ts +++ b/src/lib/target-url.spec.ts @@ -210,6 +210,31 @@ describe('assertNotLocal — IPv6 hardening (SSRF bypass guard)', () => { }); }); +describe('assertNotLocal — trailing-dot FQDN normalization (SSRF bypass guard)', () => { + // `localhost.` is the fully-qualified form of `localhost` (RFC 6761 reserves + // both to resolve to loopback). It previously bypassed the + // `host === 'localhost'` check because the WHATWG URL parser keeps the + // trailing dot on named hosts (IP literals are dot-normalized, named hosts + // are not). + it('blocks http://localhost. (trailing-dot loopback)', () => { + expectBlocked('http://localhost.'); + }); + + it('blocks http://localhost.:8080 (trailing-dot loopback with port)', () => { + expectBlocked('http://localhost.:8080'); + }); + + it('blocks http://localhost%2e (percent-encoded trailing dot)', () => { + expectBlocked('http://localhost%2e'); + }); + + // A legitimate public FQDN with a trailing dot must still be allowed + // (no false positive from the dot strip). + it('allows https://example.com. (public FQDN with trailing dot)', () => { + expectAllowed('https://example.com.'); + }); +}); + describe('assertNotLocal — allowed public URLs', () => { it('allows https://example.com', () => { expectAllowed('https://example.com'); diff --git a/src/lib/target-url.ts b/src/lib/target-url.ts index 030fc93..895c0f0 100644 --- a/src/lib/target-url.ts +++ b/src/lib/target-url.ts @@ -41,7 +41,14 @@ export function assertNotLocal(rawUrl: string): void { throw localTargetError('target-url', 'must use http or https scheme'); } - const host = parsed.hostname.toLowerCase(); + // Normalize a single trailing dot in the hostname. `localhost.` is the + // fully-qualified form of `localhost` (RFC 6761 reserves both to resolve to + // loopback), so `http://localhost.` must be rejected just like + // `http://localhost`. Without this strip, the trailing-dot form (also + // reachable via `localhost%2e`) slips past the `host === 'localhost'` check. + // IP literals are already dot-normalized by the WHATWG URL parser, so this + // only affects named hosts. + const host = parsed.hostname.toLowerCase().replace(/\.$/, ''); // Loopback / unspecified. if (host === 'localhost' || host === '0.0.0.0') { From 33b78482203a15fc7c2fb9473d6be9f50994befd Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:43:59 +0400 Subject: [PATCH 006/117] fix(auth): preserve typed API error envelope when key verification fails (#38) --- src/commands/auth.test.ts | 40 +++++++++++++++++++++++++++++++++++++++ src/commands/auth.ts | 19 ++++++++++++++----- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index b360d2a..5207085 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -312,6 +312,46 @@ describe('runConfigure', () => { expect(capture.stderr.join('\n')).toContain('profile NOT updated'); }); + it('key-rejected error preserves the typed ApiError envelope (JSON contract)', async () => { + const { deps } = makeCapture(); + const rejectedFetch: AuthDeps['fetchImpl'] = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'API key is invalid or revoked.', + nextAction: 'Rotate your key.', + requestId: 'req_reject', + details: { reason: 'malformed' }, + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ), + ) as unknown as AuthDeps['fetchImpl']; + + // The thrown error must be an ApiError (with code, nextAction, requestId) + // — not a CLIError wrapper that drops those fields. Under --output json, + // index.ts renders ApiError as the full typed envelope; CLIError would + // render only {"error":"...string..."}, violating the JSON contract. + await expect( + runConfigure( + { profile: 'default', output: 'json', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-bad' }, + credentialsPath, + fetchImpl: rejectedFetch, + }, + ), + ).rejects.toMatchObject({ + code: 'AUTH_INVALID', + exitCode: 3, + nextAction: 'Rotate your key.', + requestId: 'req_reject', + }); + }); + // The old "run `testsprite agent install`" self-bootstrap tip was removed with // the setup consolidation — runConfigure now runs ONLY as part of `setup`, // which installs the skill itself. These guard that the tip stays gone. diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 18cd622..e3b3867 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -159,13 +159,22 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): } catch (err) { const message = err instanceof Error ? err.message : String(err); stderr(`API key rejected by ${apiUrl}: ${message} — profile NOT updated`); - const exitCode = err instanceof ApiError ? err.exitCode : 3; - // Include the resolved endpoint in the thrown message so the user knows - // which host rejected the key. This prevents the "invalid or revoked" - // message from being ambiguous when the key is valid for a different env. + // When the verification call returned a typed API error (AUTH_INVALID, + // AUTH_FORBIDDEN, etc.), re-throw it directly so `index.ts` renders the + // full typed envelope under `--output json` (code, nextAction, requestId, + // details). Previously wrapping it in CLIError discarded those fields and + // emitted a bare `{"error":"...string..."}` — violating the JSON contract. + // Augment the message with the endpoint context so text-mode users still + // see which host rejected the key. + if (err instanceof ApiError) { + err.message = `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`; + throw err; + } + // Non-ApiError (truly unexpected throws like a TypeError from a + // misconfigured fetchImpl). Exit 3 (auth family). throw new CLIError( `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`, - exitCode, + 3, ); } From 2fd8f4cb78ca1cc100b634880383a8a4270ea64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EB=8F=84=ED=9B=88?= Date: Fri, 3 Jul 2026 05:44:03 +0900 Subject: [PATCH 007/117] fix(skill-nudge): require complete Codex managed section (#90) * fix(skill-nudge): require complete Codex managed section * docs(skill-nudge): document helper contracts --------- Co-authored-by: ahndohun <19940813+ahndohun@users.noreply.github.com> --- src/lib/skill-nudge.test.ts | 11 +++++++++-- src/lib/skill-nudge.ts | 17 +++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index 34f043b..c15b09d 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { MANAGED_SECTION_BEGIN, TARGETS } from './agent-targets.js'; +import { MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, TARGETS } from './agent-targets.js'; import type { OutputMode } from './output.js'; import { SKILL_NUDGE_COMMANDS, @@ -36,10 +36,17 @@ describe('isVerifySkillInstalled', () => { it('true when AGENTS.md exists AND carries our BEGIN sentinel', () => { const existsSync = (p: string) => p.endsWith('AGENTS.md'); - const readFileSync = () => `# project\n${MANAGED_SECTION_BEGIN}\n...skill...\n`; + const readFileSync = () => + `# project\n${MANAGED_SECTION_BEGIN}\n...skill...\n${MANAGED_SECTION_END}\n`; expect(isVerifySkillInstalled('/proj', { existsSync, readFileSync })).toBe(true); }); + it('false when AGENTS.md has only the BEGIN sentinel without a complete managed section', () => { + const existsSync = (p: string) => p.endsWith('AGENTS.md'); + const readFileSync = () => `# project\n${MANAGED_SECTION_BEGIN}\n...partial skill...\n`; + expect(isVerifySkillInstalled('/proj', { existsSync, readFileSync })).toBe(false); + }); + it('false when only a bare AGENTS.md (no sentinel) exists', () => { const existsSync = (p: string) => p.endsWith('AGENTS.md'); const readFileSync = () => '# my project\nNothing TestSprite here.\n'; diff --git a/src/lib/skill-nudge.ts b/src/lib/skill-nudge.ts index 1afc863..0f67572 100644 --- a/src/lib/skill-nudge.ts +++ b/src/lib/skill-nudge.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { MANAGED_SECTION_BEGIN, TARGETS } from './agent-targets.js'; +import { MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, TARGETS } from './agent-targets.js'; import { defaultCredentialsPath, readProfile } from './credentials.js'; import type { OutputMode } from './output.js'; @@ -44,8 +44,8 @@ export interface SkillPresenceDeps { * True if the `testsprite-verify` skill is installed for ANY supported agent in * `dir`. own-file targets (claude/cursor/cline/antigravity): the landing file * exists. managed-section target (codex / AGENTS.md): the file exists AND - * carries our BEGIN sentinel — a user-authored AGENTS.md without the sentinel - * does NOT count as our skill. + * carries one complete managed section — a user-authored AGENTS.md without the + * sentinels, or with a truncated section, does NOT count as our skill. * * The TARGETS table is the single source of truth for landing paths, so this * stays in lockstep with `agent install` without re-listing paths. Best-effort: @@ -59,7 +59,7 @@ export function isVerifySkillInstalled(dir: string, deps: SkillPresenceDeps = {} if (!exists(full)) continue; if (spec.mode === 'managed-section') { try { - if (read(full).includes(MANAGED_SECTION_BEGIN)) return true; + if (hasCompleteManagedSection(read(full))) return true; } catch { // unreadable AGENTS.md → treat this target as absent, keep checking } @@ -70,6 +70,14 @@ export function isVerifySkillInstalled(dir: string, deps: SkillPresenceDeps = {} return false; } +/** True when a managed-section file contains an ordered TestSprite BEGIN/END pair. */ +function hasCompleteManagedSection(content: string): boolean { + const begin = content.indexOf(MANAGED_SECTION_BEGIN); + if (begin === -1) return false; + const end = content.indexOf(MANAGED_SECTION_END, begin + MANAGED_SECTION_BEGIN.length); + return end !== -1; +} + export interface SkillNudgeContext { /** Full command path, e.g. "test run" / "auth whoami". */ commandPath: string; @@ -133,6 +141,7 @@ export function maybeEmitSkillNudge(ctx: SkillNudgeContext): void { } } +/** Interpret common env-var spellings for an enabled opt-out flag. */ function isTruthyEnv(v: string | undefined): boolean { if (v === undefined) return false; const s = v.trim().toLowerCase(); From a524702e75b24d563b351b9a1c6c9aa70d45796b Mon Sep 17 00:00:00 2001 From: nopp Date: Fri, 3 Jul 2026 03:44:08 +0700 Subject: [PATCH 008/117] fix: align documented Node engine floor (#84) --- CONTRIBUTING.md | 2 +- DOCUMENTATION.md | 2 +- README.md | 6 +++--- package-lock.json | 2 +- package.json | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b995b9..2110f0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ the thread or in Discord is welcome. ## Prerequisites -- Node 20 or newer (development happens on Node 22). +- Node 20.19+, Node 22.13+, or Node 24+ (development happens on Node 22). ## Build from source diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index a0fa9a4..1806d02 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -35,7 +35,7 @@ Or run it without installing: npx @testsprite/testsprite-cli --version ``` -Requires **Node.js ≥ 20**. +Requires **Node.js 20.19+**, **22.13+**, or **24+**. Confirm the binary works **without** configuring an API key: diff --git a/README.md b/README.md index 2d90012..067a620 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ AI ships code in minutes — verifying it hasn't. `testsprite` opens your live a

npm version npm downloads - Node >= 20 + Node 20.19+, 22.13+, or 24+ License Apache 2.0 CI

@@ -54,7 +54,7 @@ If you find `testsprite` useful, a GitHub Star ⭐️ would be greatly appreciat ## Quickstart -Requires **Node.js ≥ 20**. (No global install? `npx @testsprite/testsprite-cli` works too.) +Requires **Node.js 20.19+**, **22.13+**, or **24+**. (No global install? `npx @testsprite/testsprite-cli` works too.) ```bash npm install -g @testsprite/testsprite-cli @@ -173,7 +173,7 @@ That's the point of all of this: you no longer need the biggest, most expensive ## Contributing -Contributions are welcome — the CLI is plain TypeScript/Node (≥ 20), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: +Contributions are welcome — the CLI is plain TypeScript/Node (20.19+, 22.13+, or 24+), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: ```bash git clone https://github.com/TestSprite/testsprite-cli.git diff --git a/package-lock.json b/package-lock.json index 4ea700c..456c498 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,7 @@ "vitest": "^2.1.4" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@ampproject/remapping": { diff --git a/package.json b/package.json index 2037d26..63c5aac 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "test:e2e": "npm run build && vitest run -c vitest.e2e.config.ts" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "publishConfig": { "access": "public" From 497bf62e3e444f8e0d8e4eda15a8b9a275eb2809 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:45:06 -0500 Subject: [PATCH 009/117] fix(pagination): reject fractional pagination flags (#40) --- src/lib/pagination.test.ts | 8 ++++++++ src/lib/pagination.ts | 10 +++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/lib/pagination.test.ts b/src/lib/pagination.test.ts index 33fd735..a11ea33 100644 --- a/src/lib/pagination.test.ts +++ b/src/lib/pagination.test.ts @@ -43,10 +43,18 @@ describe('validatePaginationFlags', () => { expect(() => validatePaginationFlags({ pageSize: Number.NaN })).toThrow(ApiError); }); + it('rejects fractional pageSize values', () => { + expect(() => validatePaginationFlags({ pageSize: 1.5 })).toThrow(ApiError); + }); + it('rejects maxItems=0', () => { expect(() => validatePaginationFlags({ maxItems: 0 })).toThrow(ApiError); }); + it('rejects fractional maxItems values', () => { + expect(() => validatePaginationFlags({ maxItems: 2.5 })).toThrow(ApiError); + }); + it('accepts pageSize=100 (the hard cap)', () => { expect(() => validatePaginationFlags({ pageSize: 100 })).not.toThrow(); }); diff --git a/src/lib/pagination.ts b/src/lib/pagination.ts index 59dbc12..ba63f9e 100644 --- a/src/lib/pagination.ts +++ b/src/lib/pagination.ts @@ -27,9 +27,9 @@ const DEFAULT_PAGE_SIZE = 25; * Validates and normalizes pagination flags. Per the CLI OpenAPI spec * §components.parameters.PageSize the hard cap is 100. Values above the * cap are now rejected with exit 5 rather than silently clamped, giving - * callers fast feedback that their flag value is out of range. Sub-1 / - * NaN values also throw. `maxItems` is validated but not capped (it is a - * client-side cursor, not a server parameter). + * callers fast feedback that their flag value is out of range. Fractional, + * sub-1, and NaN values also throw. `maxItems` is validated but not capped + * (it is a client-side cursor, not a server parameter). * * NOTE: `runResultHistory` previously did its own silent clamp via * `Math.min(Math.max(1, n), 100)` — that was unified to this path by the @@ -38,7 +38,7 @@ const DEFAULT_PAGE_SIZE = 25; export function validatePaginationFlags(flags: PaginationFlags): PaginationFlags { const out: PaginationFlags = { ...flags }; if (out.pageSize !== undefined) { - if (!Number.isFinite(out.pageSize) || out.pageSize < 1) { + if (!Number.isFinite(out.pageSize) || !Number.isInteger(out.pageSize) || out.pageSize < 1) { throw localValidationError( 'page-size', `must be a positive integer between 1 and ${HARD_PAGE_SIZE_CAP}`, @@ -52,7 +52,7 @@ export function validatePaginationFlags(flags: PaginationFlags): PaginationFlags } } if (out.maxItems !== undefined) { - if (!Number.isFinite(out.maxItems) || out.maxItems < 1) { + if (!Number.isFinite(out.maxItems) || !Number.isInteger(out.maxItems) || out.maxItems < 1) { throw localValidationError('maxItems', 'must be a positive integer'); } } From 9aa7ba9d6767b64234b28de044cec76bf15aee6e Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:45:13 +0400 Subject: [PATCH 010/117] fix(project): reject empty/whitespace-only --name in create and update (#36) project create/update validated --name with the action handler's if (!name) check, which a whitespace-only string passes (a non-empty string is truthy). The blank name was then sent verbatim, creating a junk-named project. The sibling est create already rejects this via the requireString whitespace guard (dogfood P1 fix #1); this aligns project create/update with that behavior. Adds 2 regression tests. --- src/commands/project.test.ts | 52 ++++++++++++++++++++++++++++++++++++ src/commands/project.ts | 12 +++++++++ 2 files changed, 64 insertions(+) diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 057296a..c928a86 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -569,6 +569,33 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + + it('rejects a whitespace-only --name with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network — validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: ' ', + targetUrl: 'https://example.com', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); }); // --------------------------------------------------------------------------- @@ -646,6 +673,31 @@ describe('runUpdate', () => { expect(fetchImpl).not.toHaveBeenCalled(); }); + it('rejects a whitespace-only --name with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + name: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('P7 — dry-run returns canned shape without network call', async () => { resetDryRunBannerForTesting(); const { credentialsPath } = makeCreds(); diff --git a/src/commands/project.ts b/src/commands/project.ts index 41abab1..74d621e 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -141,6 +141,15 @@ export async function runCreate( // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. assertIdempotencyKey(opts.idempotencyKey); + // Reject empty / whitespace-only names so a junk record never reaches the + // backend — matches the `requireString` whitespace guard `test create` uses + // (dogfood P1 fix #1). Without this, `--name " "` passes the action + // handler's `if (!name)` check (a non-empty string is truthy) and is sent + // verbatim, creating a blank-named project. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); + } + // P1-3: client-side length checks matching server limits. if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); @@ -251,6 +260,9 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } From 21dc760f1bcc898d300ba5c4d873ad807cdde7b8 Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:45:24 +0100 Subject: [PATCH 011/117] fix(cli): validate --output uniformly across all command groups (#14) Only `test` and `project` validated the global `--output` flag. The `auth`, `usage`, `agent`, and `init` command groups resolved it with `globals.output ?? 'text'`, so an unrecognised value (e.g. a typo like `--output josn`) was silently coerced to text mode instead of being rejected. A coding agent that asked for `--output json` then received a human-readable text payload and failed to parse it as JSON, with no signal as to why. Extract the validation into a shared `resolveOutputMode` helper in `lib/output.js` and route every command group's `resolveCommonOptions` through it. Invalid values now throw a typed VALIDATION_ERROR (exit 5) with an actionable message everywhere. This also unifies the error wording, which previously differed between `test` ("Flag `--output` is invalid: must be one of: json, text.") and `project` ("--output must be one of: json, text"). --- src/commands/agent.ts | 4 ++-- src/commands/auth.ts | 4 ++-- src/commands/init.ts | 4 ++-- src/commands/project.ts | 8 ++------ src/commands/test.ts | 8 ++------ src/commands/usage.ts | 4 ++-- src/lib/output.test.ts | 33 ++++++++++++++++++++++++++++++++- src/lib/output.ts | 22 ++++++++++++++++++++++ test/cli.subprocess.test.ts | 27 +++++++++++++++++++++++++++ 9 files changed, 93 insertions(+), 21 deletions(-) diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 3347453..0c6c6c1 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -4,7 +4,7 @@ import { Command } from 'commander'; import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; import { CLIError, localValidationError } from '../lib/errors.js'; import type { OutputMode } from '../lib/output.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptText } from '../lib/prompt.js'; import { type AgentTarget, @@ -852,7 +852,7 @@ function resolveCommonOptions(command: Command): CommonOptions { const globals = command.optsWithGlobals() as Partial; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/auth.ts b/src/commands/auth.ts index e3b3867..bd6dc13 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -18,7 +18,7 @@ import { import { loadConfig } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptSecret } from '../lib/prompt.js'; export interface MeResponse { @@ -328,7 +328,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/init.ts b/src/commands/init.ts index 6a432c2..7615a44 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -14,7 +14,7 @@ import { Command } from 'commander'; import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import type { AuthDeps, MeResponse } from './auth.js'; import { runConfigure, runWhoami } from './auth.js'; import type { AgentDeps, AgentFs, InstallResult } from './agent.js'; @@ -424,7 +424,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/project.ts b/src/commands/project.ts index 74d621e..2eaa416 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -9,7 +9,7 @@ import { import { ApiError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { assertNotLocal } from '../lib/target-url.js'; import { assertIdempotencyKey } from '../lib/validate.js'; import { @@ -506,13 +506,9 @@ function resolveCommonOptions(command: Command): CommonOptions { requestTimeout?: string; }; // P2-8: validate --output before allowing silent fallback to 'text'. - const rawOutput = globals.output; - if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { - throw localValidationError('--output must be one of: json, text'); - } return { profile: globals.profile ?? 'default', - output: (globals.output as OutputMode | undefined) ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/test.ts b/src/commands/test.ts index c61e84b..62a14ec 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -34,7 +34,7 @@ import { import { REQUEST_TIMEOUT_DEFAULT_MS, REQUEST_TIMEOUT_MAX_MS } from '../lib/http.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { fetchSinglePage, paginate, @@ -7809,13 +7809,9 @@ function resolveCommonOptions(command: Command): CommonOptions { // P2-8: validate --output before allowing silent fallback to 'text'. // An invalid value (e.g. `--output yaml`) must exit 5 with a clear error // rather than silently treating the request as text mode. - const rawOutput = globals.output; - if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { - throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); - } return { profile: globals.profile ?? 'default', - output: (globals.output as OutputMode | undefined) ?? 'text', + output: resolveOutputMode(globals.output), dryRun: globals.dryRun ?? false, endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, diff --git a/src/commands/usage.ts b/src/commands/usage.ts index d8a0f21..a539261 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -22,7 +22,7 @@ import { import { loadConfig } from '../lib/config.js'; import { resolvePortalBase } from '../lib/facade.js'; import type { FetchImpl } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; /** * Usage/balance response from `/me` (when the backend supplies it) or a future @@ -216,7 +216,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/lib/output.test.ts b/src/lib/output.test.ts index b572bda..bab3b53 100644 --- a/src/lib/output.test.ts +++ b/src/lib/output.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { Output, isOutputMode } from './output.js'; +import { Output, isOutputMode, resolveOutputMode } from './output.js'; +import { ApiError } from './errors.js'; describe('isOutputMode', () => { it('accepts json and text', () => { @@ -15,6 +16,36 @@ describe('isOutputMode', () => { }); }); +describe('resolveOutputMode', () => { + it('returns the mode verbatim for valid values', () => { + expect(resolveOutputMode('json')).toBe('json'); + expect(resolveOutputMode('text')).toBe('text'); + }); + + it('defaults to text when the flag is omitted (undefined)', () => { + expect(resolveOutputMode(undefined)).toBe('text'); + }); + + it('throws a typed VALIDATION_ERROR (exit 5) instead of silently falling back to text', () => { + // The footgun this guards against: an agent that asks for `--output json` + // but mistypes it would otherwise receive a text payload and fail to parse + // it as JSON with no signal. Every command group must reject, not coerce. + for (const bad of ['josn', 'yaml', 'JSON', 'Text', '']) { + let caught: unknown; + try { + resolveOutputMode(bad); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('must be one of: json, text'); + } + }); +}); + describe('Output', () => { let logSpy: ReturnType; let errorSpy: ReturnType; diff --git a/src/lib/output.ts b/src/lib/output.ts index 79b3f11..4d59cca 100644 --- a/src/lib/output.ts +++ b/src/lib/output.ts @@ -1,3 +1,5 @@ +import { localValidationError } from './errors.js'; + export type OutputMode = 'json' | 'text'; /** @@ -13,6 +15,26 @@ export function isOutputMode(value: unknown): value is OutputMode { return value === 'json' || value === 'text'; } +/** + * Resolve a raw `--output` flag value to a concrete {@link OutputMode}. + * + * `undefined` (flag omitted) resolves to the default `'text'`. Any other + * value that is not `'json'` or `'text'` throws a typed VALIDATION_ERROR + * (exit 5) with an actionable message. + * + * The alternative — silently falling back to `'text'` — is a footgun for the + * CLI's primary consumer (coding agents): a caller that asks for + * `--output json` but mistypes it (`--output josn`) would otherwise receive a + * human-readable text payload and fail to parse it as JSON, with no signal as + * to why. Every command group routes its global-option resolution through this + * helper so the validation is uniform. + */ +export function resolveOutputMode(raw: unknown): OutputMode { + if (raw === undefined) return 'text'; + if (isOutputMode(raw)) return raw; + throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); +} + export interface OutputStreams { /** * Line-oriented stdout writer. Each call is one logical line; the diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 21d7cfe..0f05bf7 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -530,6 +530,33 @@ describe('malformed --endpoint-url is rejected (exit 5), not retried as a networ }, 30_000); }); +describe('invalid --output is rejected uniformly (exit 5)', () => { + // Regression: previously only `test` and `project` validated `--output`; + // `auth`, `usage`, `agent`, and `init` silently coerced an unknown value to + // text mode. An agent that asked for `--output json` but mistyped it then + // received a text payload it could not parse, with no signal as to why. Every + // command group now routes through resolveOutputMode (exit 5 on bad input). + + // Note: when `--output` itself is the invalid value, the requested mode is + // unusable for the error envelope, so it is rendered in text mode (the catch + // block in index.ts falls back to text for an unrecognised --output). + + it('agent list --output josn exits 5 with an actionable message (offline command)', async () => { + const result = await runCli(['--output', 'josn', 'agent', 'list'], {}); + expect(result.exitCode).toBe(5); + expect(result.stderr).toContain('must be one of: json, text'); + }, 30_000); + + it('auth status --output yaml exits 5 before any network call', async () => { + const result = await runCli(['--output', 'yaml', 'auth', 'status'], { + TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_URL: baseUrl, + }); + expect(result.exitCode).toBe(5); + expect(result.stderr).toContain('must be one of: json, text'); + }, 30_000); +}); + describe('a malformed --profile is rejected (exit 5), not silently corrupting credentials', () => { // A profile name becomes an INI section header (`[name]`). `prod]` would // serialise to `[prod]]`, which the parser cannot read back — `setup` would From 000c196375883e2df773a55c53df79ff7f67412f Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:45:32 +0400 Subject: [PATCH 012/117] ci: add Node 20 to test and build matrix (#133) --- .github/workflows/ci.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 890f9c7..ad3ef6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,15 +48,18 @@ jobs: - run: npm run typecheck test: - name: Unit Tests + name: Unit Tests (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: - node-version: 22 + node-version: ${{ matrix.node-version }} cache: 'npm' - run: npm ci @@ -65,15 +68,18 @@ jobs: CI: true build: - name: Build + name: Build (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: - node-version: 22 + node-version: ${{ matrix.node-version }} cache: 'npm' - run: npm ci From 5e0f7c4db423885eb2b442e9627643bd6f8c6199 Mon Sep 17 00:00:00 2001 From: Sahil Rakhaiya <144577420+SahilRakhaiya05@users.noreply.github.com> Date: Fri, 3 Jul 2026 02:15:39 +0530 Subject: [PATCH 013/117] fix(test): reject empty code get --out and strip code-file BOM (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(test): reject empty code get --out and strip code-file BOM When test code get --out receives an empty inline body, closeOutputFile left a zero-byte file with exit 0 — scripts and agents treated that as a successful download. Abort the sink, unlink any artifact, and surface VALIDATION_ERROR instead. Plan/steps JSON already strip a leading UTF-8 BOM from PowerShell 5.1 files; apply the same strip to --code-file reads so uploaded test source is not corrupted by an invisible U+FEFF prefix. * fix(test): rebase onto v0.2.0 and harden empty-code --out cleanup - Resolve rebase conflicts: keep atomic temp-file --out writes from main while rejecting empty inline code with VALIDATION_ERROR (exit 5) - abortOutputFile: wait for stream close before unlinking tmpPath - BOM test: use .py code-file (assertPythonCodeFile from v0.2.0) - CI: build before test + fileParallelism false (dist/ race flake) --- .github/workflows/ci.yml | 1 + src/commands/test.test.ts | 80 ++++++++++++++++++++++++++++++++++++--- src/commands/test.ts | 34 ++++++++++++++--- vitest.config.ts | 3 ++ 4 files changed, 107 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad3ef6e..edde076 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,7 @@ jobs: cache: 'npm' - run: npm ci + - run: npm run build - run: npm test env: CI: true diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 4d205e0..49127bd 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -1591,22 +1591,66 @@ describe('runCodeGet', () => { expect(leftovers).toEqual([]); }); - // Regression: the "no code generated yet" branch writes nothing but - // previously still closed (and thus truncated) the opened file. + it('--out (text mode) rejects empty inline code with VALIDATION_ERROR and leaves no artifact', async () => { + const { credentialsPath } = makeCreds(); + const emptyCode: CliTestCode = { ...TEST_CODE_INLINE, code: '' }; + const fetchImpl = makeFetch(() => ({ body: emptyCode })); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-empty-out-')); + const target = join(dir, 'empty.ts'); + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: target, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'out' }), + }); + expect(existsSync(target)).toBe(false); + }); + + // Regression: empty inline code with --out must reject (exit 5) without + // truncating or replacing a pre-existing destination file. it('--out: "no code generated yet" leaves a pre-existing file untouched', async () => { const { credentialsPath } = makeCreds(); const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-empty-')); const target = join(dir, 'existing.ts'); writeFileSync(target, 'PRE-EXISTING CONTENT', 'utf8'); const fetchImpl = makeFetch(() => ({ body: { ...TEST_CODE_INLINE, code: '' } })); - await runCodeGet( - { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: target }, - { credentialsPath, fetchImpl, stderr: () => undefined }, - ); + await expect( + runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: target }, + { credentialsPath, fetchImpl, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'out' }), + }); expect(readFileSync(target, 'utf-8')).toBe('PRE-EXISTING CONTENT'); const leftovers = readdirSync(dir).filter(f => f !== 'existing.ts'); expect(leftovers).toEqual([]); }); + + it('text mode without --out still hints on stderr when inline code is empty', async () => { + const { credentialsPath } = makeCreds(); + const emptyCode: CliTestCode = { ...TEST_CODE_INLINE, code: '' }; + const fetchImpl = makeFetch(() => ({ body: emptyCode })); + const stderr: string[] = []; + const got = await runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stderr: line => stderr.push(line) }, + ); + expect(got.code).toBe(''); + expect(stderr.join('\n')).toContain('no code generated yet'); + }); }); describe('runCodePut', () => { @@ -1661,6 +1705,30 @@ describe('runCodePut', () => { expect(sent.headers.get('content-type')).toBe('application/json'); }); + it('strips a UTF-8 BOM from --code-file before uploading (Windows PowerShell 5.1 default)', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p4-bom-')); + const codeFile = join(dir, 'updated.py'); + writeFileSync(codeFile, '\uFEFF' + 'updated body', 'utf8'); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenBody).toEqual({ code: 'updated body' }); + }); + it('forwards --language in the body when set', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('print("hi")'); diff --git a/src/commands/test.ts b/src/commands/test.ts index 62a14ec..7e6c94f 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -911,7 +911,7 @@ function readCodeFileGuarded(path: string): string { function readCodeFile(path: string): string { try { - return readFileSync(resolveAbsolute(path), 'utf8'); + return stripBom(readFileSync(resolveAbsolute(path), 'utf8')); } catch (err) { const code = (err as NodeJS.ErrnoException).code; if (code === 'ENOENT') { @@ -3292,7 +3292,7 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro return code; } - const fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; + let fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; const out = fileSink ? makeFileOutput(opts.output, fileSink) : makeOutput(opts.output, deps); const client = makeClient(opts, deps); @@ -3317,9 +3317,20 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro } else if (code.code === '' || code.code === null) { // P2-10: draft test with no code yet — empty body would produce // silent empty stdout. Print a friendly hint to stderr instead so - // the operator knows what happened, and keep exit 0. Nothing was - // written, so the temp file is discarded below without touching - // a pre-existing `--out` file. + // the operator knows what happened, and keep exit 0 when no `--out`. + // + // With `--out`, refuse to leave a zero-byte artifact behind: agents + // and scripts that check file size would otherwise treat exit 0 as + // a successful download. Discard the temp sink without touching a + // pre-existing destination file. + if (fileSink) { + await abortOutputFile(fileSink); + fileSink = null; + throw localValidationError( + 'out', + 'test has no generated code yet — run the test first (refusing to write an empty --out file)', + ); + } const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); stderrFn('(no code generated yet — run the test first)'); } else { @@ -7997,6 +8008,19 @@ async function closeOutputFile(sink: FileSink, commit: boolean): Promise { await rename(sink.tmpPath, sink.path); } +/** Tear down an opened `--out` sink without leaving a zero-byte artifact. */ +async function abortOutputFile(sink: FileSink): Promise { + await new Promise(resolve => { + if (sink.stream.destroyed) { + resolve(); + return; + } + sink.stream.once('close', () => resolve()); + sink.stream.destroy(); + }); + await unlink(sink.tmpPath).catch(() => undefined); +} + /** A presigned `code` body is any `https://` URL — never anything else. */ export function isPresignedCodeUrl(code: string): boolean { return code.startsWith('https://'); diff --git a/vitest.config.ts b/vitest.config.ts index 18ea00f..add8f0e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,9 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], + // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel + // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). + fileParallelism: false, coverage: { provider: 'v8', reporter: ['text', 'text-summary', 'json-summary', 'html'], From dfdee5a3e051987c0ce090512c6d197532f4deb1 Mon Sep 17 00:00:00 2001 From: Chara Date: Thu, 2 Jul 2026 22:45:45 +0200 Subject: [PATCH 014/117] fix(prompt): preserve buffered input between prompts (#118) Co-authored-by: cmdr-chara <249489759+cmdr-chara@users.noreply.github.com> --- src/lib/prompt.test.ts | 35 ++++++++++++++++++++++++++++ src/lib/prompt.ts | 53 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/src/lib/prompt.test.ts b/src/lib/prompt.test.ts index a552882..45275e3 100644 --- a/src/lib/prompt.test.ts +++ b/src/lib/prompt.test.ts @@ -39,6 +39,31 @@ describe('promptText', () => { expect(await promptText('? ', { input, output })).toBe('ok'); }); + it('preserves buffered answers for sequential prompts on the same stream', async () => { + const input = Readable.from(['first\nsecond\nthird\n']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + await expect(promptText('Three: ', { input, output })).resolves.toBe('third'); + }); + + it('uses buffered tail input at EOF for a following prompt', async () => { + const input = Readable.from(['first\nsecond']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + }); + + it('preserves buffered CRLF answers for sequential prompts', async () => { + const input = Readable.from(['first\r\nsecond\r\n']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + }); + it('returns the buffered input on stream end without newline', async () => { const input = Readable.from(['eof-no-newline']); const output = new CaptureStream(); @@ -70,6 +95,16 @@ describe('promptSecret (non-TTY behavior)', () => { expect(await promptSecret('? ', { input, output })).toBe('abd'); }); + it('preserves buffered secret answers for sequential prompts', async () => { + const input = Readable.from(['sk-one\nsk-two\n']); + const output = new CaptureStream(); + + await expect(promptSecret('First key: ', { input, output })).resolves.toBe('sk-one'); + await expect(promptSecret('Second key: ', { input, output })).resolves.toBe('sk-two'); + expect(output.text()).not.toContain('sk-one'); + expect(output.text()).not.toContain('sk-two'); + }); + it('rejects on Ctrl-C input', async () => { const ETX = String.fromCharCode(0x03); const input = Readable.from([`abc${ETX}`]); diff --git a/src/lib/prompt.ts b/src/lib/prompt.ts index 96fca29..3561b0a 100644 --- a/src/lib/prompt.ts +++ b/src/lib/prompt.ts @@ -11,6 +11,8 @@ interface RawModeCapable { resume?: () => unknown; } +const pendingPromptInput = new WeakMap(); + export async function promptText(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; const output = streams.output ?? process.stdout; @@ -41,18 +43,29 @@ function readLine( return new Promise((resolve, reject) => { let buffer = ''; let resolved = false; + let listening = false; const onData = (chunk: Buffer | string): void => { const str = typeof chunk === 'string' ? chunk : chunk.toString('utf-8'); - for (const ch of str) { - const code = ch.charCodeAt(0); + processText(str); + }; + + const processText = (str: string): void => { + for (let i = 0; i < str.length; i += 1) { + const code = str.charCodeAt(i); // Enter (CR or LF) if (code === 13 || code === 10) { + let nextIndex = i + 1; + if (code === 13 && str.charCodeAt(nextIndex) === 10) { + nextIndex += 1; + } + savePendingInput(str.slice(nextIndex)); finish(); return; } // Ctrl-C if (code === 3) { + savePendingInput(''); cleanup(); output.write('\n'); if (!resolved) { @@ -71,7 +84,7 @@ function readLine( } // Drop other control chars if (code < 32) continue; - buffer += ch; + buffer += str[i]; if (mask) output.write('*'); } }; @@ -89,9 +102,12 @@ function readLine( }; const cleanup = (): void => { - input.off('data', onData); - input.off('end', onEnd); - input.off('error', onError); + if (listening) { + input.off('data', onData); + input.off('end', onEnd); + input.off('error', onError); + listening = false; + } const pausable = input as { pause?: () => unknown }; if (typeof pausable.pause === 'function') pausable.pause(); }; @@ -105,12 +121,37 @@ function readLine( } }; + const savePendingInput = (text: string): void => { + if (text.length > 0) { + pendingPromptInput.set(input, text); + } else { + pendingPromptInput.delete(input); + } + }; + + const pending = pendingPromptInput.get(input); + if (pending !== undefined) { + pendingPromptInput.delete(input); + processText(pending); + if (resolved) return; + if (isInputEnded(input)) { + finish(); + return; + } + } + input.on('data', onData); input.on('end', onEnd); input.on('error', onError); + listening = true; const resumable = input as { resume?: () => unknown }; if (typeof resumable.resume === 'function') resumable.resume(); }); } +function isInputEnded(input: NodeJS.ReadableStream): boolean { + const state = input as { readableEnded?: boolean; destroyed?: boolean; closed?: boolean }; + return state.readableEnded === true || state.destroyed === true || state.closed === true; +} + export type { Writable }; From be9784a639931fb602b00cb00624793db35f6557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EB=8F=84=ED=9B=88?= Date: Fri, 3 Jul 2026 05:45:52 +0900 Subject: [PATCH 015/117] fix(test): emit auto-minted idempotency-key under --output json in rerun and run --all (#128) test run, test create, create-batch, plan put, code put, update, and delete all print the auto-minted idempotency key to stderr under --output json (as well as --verbose / --debug) so JSON-mode automation can capture the key and replay a retry safely. test rerun and test run --all minted a key but only echoed it under --debug / --verbose, so CI flows using --output json silently lost it. Align both paths with the shared guard used by every other minting site, and cover the JSON-mode emission (and the text-mode silence) with regression tests. Co-authored-by: ahndohun <19940813+ahndohun@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- DOCUMENTATION.md | 4 +- src/commands/test.rerun.spec.ts | 68 +++++++++++++++++++++++++++++++++ src/commands/test.run.spec.ts | 33 ++++++++++++++++ src/commands/test.ts | 10 +---- 4 files changed, 105 insertions(+), 10 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 1806d02..700671b 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -346,7 +346,7 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ testsprite test run test_xxxxxxxx --dry-run --output json ``` -`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr at `--verbose`); pass `--idempotency-key ` to control it explicitly. +`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. #### `testsprite test rerun [test-id...]` @@ -376,7 +376,7 @@ Flags: - `--auto-heal` / `--no-auto-heal` — frontend AI heal-on-drift, **on by default** for FE reruns; opt out with `--no-auto-heal`. Verbatim-replay passes are free; a heal engage costs a small amount of credit. Ignored for backend tests. - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. -- `--idempotency-key ` — auto-minted when omitted. +- `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 86c8e89..acd5b02 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -1637,6 +1637,74 @@ describe('--idempotency-key passthrough', () => { expect(receivedKey).toBe('my-custom-key-abc'); }); + + it('emits the auto-minted idempotency-key on stderr in JSON output mode (parity with test run)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + + it('does NOT emit an idempotency-key line in default text mode', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'text', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines.some(l => l.includes('idempotency-key:'))).toBe(false); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 7e2b55e..2de9f9c 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -3470,6 +3470,39 @@ describe('dashboardUrl on run completion', () => { ); }); + it('run --all: emits the auto-minted idempotency-key on stderr in JSON output mode (parity with test run)', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_f_01', enqueuedAt: '2026-06-10T10:00:00.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const stderrLines: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: batchResp })), + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + it('run --all --wait (prod endpoint): summary items carry dashboardUrl + stderr Dashboard line', async () => { const { credentialsPath } = makeCreds('sk-user-test', PROD_API); const batchResp: BatchRunFreshResponse = { diff --git a/src/commands/test.ts b/src/commands/test.ts index 7e6c94f..a9ff9e3 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5150,12 +5150,9 @@ export async function runTestRunAll( }; const idempotencyKey = opts.idempotencyKey ?? `cli-batch-run-fresh-${randomUUID()}`; - if (opts.idempotencyKey === undefined && opts.debug) { + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderrFn(`idempotency-key: ${idempotencyKey}`); } - if (opts.idempotencyKey === undefined && opts.verbose) { - stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); - } // Resolve testIds: fetch all BE tests in the project, apply --filter. let testIds: string[] | undefined; @@ -5718,12 +5715,9 @@ export async function runTestRerun( // slow rerun trigger / long-poll under load isn't cut at the 120s default. const client = makeClient({ ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs(opts) }, deps); const idempotencyKey = opts.idempotencyKey ?? `cli-rerun-${randomUUID()}`; - if (opts.idempotencyKey === undefined && opts.debug) { + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderrFn(`idempotency-key: ${idempotencyKey}`); } - if (opts.idempotencyKey === undefined && opts.verbose) { - stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); - } // ------------------------------------------------------------------------- // Single rerun path From 03c4f11ae8ef14251299863d521e49d1e4f10a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EB=8F=84=ED=9B=88?= Date: Fri, 3 Jul 2026 05:45:59 +0900 Subject: [PATCH 016/117] fix(agent): apply symlink fail-close guard to own-file --dry-run installs (#129) The codex managed-section branch already runs inspectTargetPath during --dry-run ([P2]) so a planted symlink is refused the same way the real install refuses it. The own-file targets (claude, cursor, cline, antigravity) skipped that guard in dry-run and reported the write as successful, so 'agent install --dry-run' could claim success for an install that would actually exit 5. Run the same guard in the own-file dry-run branch and cover both the symlinked-target and symlinked-parent cases with regression tests. Co-authored-by: ahndohun <19940813+ahndohun@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- src/commands/agent.test.ts | 44 ++++++++++++++++++++++++++++++++++++++ src/commands/agent.ts | 11 ++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 523c5fc..9bdbdf1 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -1311,6 +1311,50 @@ describe('runInstall — symlink safety', () => { expect(writeCalls.length).toBe(0); // never wrote a .bak nor through the link }); + it('dry-run: refuses (exit 5) when the target file is a symlink (parity with real install)', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + // Same planted SKILL.md symlink as the real-install case above: dry-run + // must report the same refusal the real install would, not a success. + seedSymlink(path.resolve(CWD, TARGETS.claude.path)); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false, dryRun: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + + it('dry-run: refuses (exit 5) when a parent path component is a symlink', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + seedSymlink(path.resolve(CWD, '.claude')); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false, dryRun: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + it('does not write through a symlinked .bak slot — backs up to a numbered slot', async () => { const { store, fs: agentFs, seedFile, seedSymlink } = makeMemFs(); const abs = path.resolve(CWD, TARGETS.claude.path); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 0c6c6c1..ebde317 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -638,6 +638,17 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr const content = renderForTarget(t, skill, bodyForSkill(skill)).content; if (opts.dryRun) { + // Apply the SAME symlink fail-close guard as the real install path + // below (the codex managed-section branch already does this). Without + // it, dry-run reports success for a planted symlink that the real + // install would refuse with exit 5. + const dryRunSt = await inspectTargetPath(agentFs, root, relPath); + if (dryRunSt !== null && !dryRunSt.isFile) { + throw new CLIError( + `${relPath} exists but is not a regular file — remove it and re-run.`, + 5, + ); + } const bytes = Buffer.byteLength(content, 'utf8'); dryRunLines.push({ abs, bytes, note: '' }); results.push({ target: t, path: relPath, action: 'dry-run', skills: [skill] }); From 03251d744a0d5324eac177f2c42a1eb7e7f3277b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EB=8F=84=ED=9B=88?= Date: Fri, 3 Jul 2026 05:46:05 +0900 Subject: [PATCH 017/117] fix(test): stop run --all --wait from polling queued runs past the shared deadline (#130) runTestRunAll computes each member's poll budget as Math.max(1, ceil(batchDeadlineMs - now)), so a run whose turn arrives after the shared --timeout deadline still gets a fresh >=1s poll. With --max-concurrency bounding the fan-out, a batch could overshoot the documented shared deadline and report a late 'passed' for a member that should have been reported as 'timeout' (exit 7). Guard the poll helper the same way the create-batch --run --wait path already does: if the shared deadline is exhausted before a member's poll starts, return the existing timeout-shaped member result without calling pollRunUntilTerminal. Regression test drives the clock past the deadline during the first member's poll and asserts the second member is never polled and reports 'timeout'. Co-authored-by: ahndohun <19940813+ahndohun@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- src/commands/test.run.spec.ts | 55 +++++++++++++++++++++++++++++++++++ src/commands/test.ts | 15 +++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 2de9f9c..0a10df5 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -2487,6 +2487,61 @@ describe('runTestRunAll — batch fresh run', () => { expect(payload.accepted.every(r => r.status === 'passed')).toBe(true); }); + it('run --all --wait: does not start a fresh poll for a queued run after the shared deadline expired', async () => { + const { credentialsPath } = makeCreds(); + const baseNow = new Date('2026-06-09T10:00:00.000Z').getTime(); + let nowMs = baseNow; + const runFetches: string[] = []; + const stdoutLines: string[] = []; + let caughtError: unknown; + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => nowMs); + + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'POST') return { body: BATCH_FRESH_RESP }; + + const runId = url.split('/runs/')[1]?.split('?')[0] ?? 'run_unknown'; + runFetches.push(runId); + if (runId === 'run_fresh_01') { + nowMs = baseNow + 2000; + return { body: makePassedRun(runId, 'test_be_01') }; + } + return { body: makePassedRun(runId, 'test_be_02') }; + }); + + try { + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 1, + maxConcurrency: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + } catch (err) { + caughtError = err; + } finally { + nowSpy.mockRestore(); + } + + const payload = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ runId: string; status: string }>; + }; + expect(runFetches).toEqual(['run_fresh_01']); + expect(payload.accepted.find(r => r.runId === 'run_fresh_02')?.status).toBe('timeout'); + expect((caughtError as { exitCode?: number } | undefined)?.exitCode).toBe(7); + }); + it('--wait with a failed run → exit 1', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch((url, init) => { diff --git a/src/commands/test.ts b/src/commands/test.ts index a9ff9e3..8489a01 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5432,7 +5432,20 @@ export async function runTestRunAll( async function pollFreshAccepted(entry: BatchRunFreshAccepted): Promise { const runId = entry.runId; - const remainingSeconds = Math.max(1, Math.ceil((batchDeadlineMs - Date.now()) / 1000)); + const remainingMs = batchDeadlineMs - Date.now(); + if (remainingMs <= 0) { + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${opts.timeoutSeconds}s`, + exitCode: 7, + }, + }; + } + const remainingSeconds = Math.ceil(remainingMs / 1000); const resolveAlternate = makeBackendWaitFallback({ client, resolveTestId: () => entry.testId, From 7b7d80db786068b50e4f44b7f200ed3556e31a1f Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:46:12 +0400 Subject: [PATCH 018/117] fix(credentials): strip CR/LF from values to prevent INI injection (#131) --- src/lib/credentials.test.ts | 32 ++++++++++++++++++++++++++++++++ src/lib/credentials.ts | 10 +++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index ac038f3..896d057 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -88,6 +88,38 @@ describe('serializeCredentials', () => { expect(text).toContain('api_key = sk'); expect(text).not.toContain('api_url'); }); + + it('strips newline characters from values to prevent INI injection', () => { + // A malicious apiUrl with embedded newlines could inject new key-value + // pairs or section headers into the credentials file. The serializer + // must strip \n and \r so the written file has exactly one value per + // field and no injected content parsed as separate keys/sections. + const malicious = 'https://evil.com\napi_key = sk-HIJACKED\n[admin]\napi_key = sk-admin'; + const text = serializeCredentials({ default: { apiKey: 'sk-real', apiUrl: malicious } }); + // The output must NOT contain a standalone [admin] section header + // (it would be on its own line if injection succeeded) + const lines = text.split('\n'); + // Only one section header exists: [default] + const sectionHeaders = lines.filter(l => /^\[.+\]$/.test(l.trim())); + expect(sectionHeaders).toEqual(['[default]']); + // Only one api_key line exists (the real one, not an injected duplicate) + const apiKeyLines = lines.filter(l => l.trim().startsWith('api_key')); + expect(apiKeyLines).toHaveLength(1); + expect(apiKeyLines[0]).toContain('sk-real'); + // Round-trip: reading back must return only the real key, not the injected one + const parsed = parseCredentials(text); + expect(parsed['default']?.apiKey).toBe('sk-real'); + expect(parsed['admin']).toBeUndefined(); + }); + + it('strips \\r\\n (CRLF) injection from values', () => { + const text = serializeCredentials({ default: { apiUrl: 'https://x.com\r\napi_key = pwned' } }); + const parsed = parseCredentials(text); + // The injected api_key must NOT be parsed as a real key + expect(parsed['default']?.apiKey).toBeUndefined(); + // The api_url value is on one line (newlines stripped) + expect(parsed['default']?.apiUrl).toContain('https://x.com'); + }); }); describe('readCredentialsFile / readProfile', () => { diff --git a/src/lib/credentials.ts b/src/lib/credentials.ts index c885c66..21e2ac0 100644 --- a/src/lib/credentials.ts +++ b/src/lib/credentials.ts @@ -116,7 +116,15 @@ export function serializeCredentials(file: CredentialsFile): string { for (const field of fields) { const value = entry[field]; if (value === undefined || value === '') continue; - lines.push(`${FIELD_TO_FILE_KEY[field]} = ${value}`); + // Guard against INI injection: a value containing newline characters + // would be serialized across multiple lines, allowing an attacker to + // inject arbitrary key-value pairs (or new section headers) into the + // credentials file. A valid API key or URL never contains \n or \r. + // Strip them so a compromised env var or MITM'd backend response + // cannot override the stored api_key on subsequent reads. + const sanitized = value.replace(/[\r\n]/g, ''); + if (sanitized === '') continue; + lines.push(`${FIELD_TO_FILE_KEY[field]} = ${sanitized}`); } lines.push(''); } From 76fff29c91c95f0900ba3ea2dd14f75e69396042 Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:48:23 +0400 Subject: [PATCH 019/117] fix(target-url): treat trailing-dot hostnames as loopback in SSRF guard (#37) assertNotLocal lowercased the hostname but did not strip a trailing dot, so http://localhost. (the FQDN form of localhost, RFC 6761) and http://localhost%2e bypassed the host === 'localhost' loopback check. IP literals are already dot-normalized by the WHATWG URL parser, so only named hosts were affected. Strips one trailing dot before the comparison. Adds 4 regression tests (3 blocked variants + 1 public-FQDN no-false-positive). From 8da07fb4b11bf003e9bc62c5d479424c663350b2 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:48:27 -0500 Subject: [PATCH 020/117] fix(project): avoid password file reads in dry-run update (#54) --- src/commands/project.test.ts | 27 ++++++++++++++++++++ src/commands/project.ts | 48 +++++++++++++++++++++++------------- test/cli.subprocess.test.ts | 16 ++++++++++++ 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index c928a86..1fcbd99 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -729,6 +729,33 @@ describe('runUpdate', () => { expect(err).toContain(DRY_RUN_BANNER); }); + it('P7 — dry-run with --password-file does not read the filesystem', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'proj_dry', + passwordFile: '/tmp/definitely-not-here-testsprite', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(result.id).toBe('proj_dry'); + expect(result.updatedFields).toContain('password'); + }); + it('P7 — renders text mode with updatedFields and updatedAt', async () => { const { credentialsPath } = makeCreds(); const updateResponse: CliUpdateProjectResponse = { diff --git a/src/commands/project.ts b/src/commands/project.ts index 2eaa416..fd34b47 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -270,27 +270,24 @@ export async function runUpdate( throw localValidationError('--description must be at most 2000 characters'); } - // Resolve password - let password = opts.password; - if (password === undefined && opts.passwordFile !== undefined) { - password = readFileSync(opts.passwordFile, 'utf8').trim(); - } - // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); } - const mutableFields: Record = { - name: opts.name, - targetUrl: opts.targetUrl, - username: opts.username, - password, - description: opts.description, - instruction: opts.instruction, + const passwordSupplied = opts.password !== undefined || opts.passwordFile !== undefined; + const mutableFields: Record = { + name: opts.name !== undefined, + targetUrl: opts.targetUrl !== undefined, + username: opts.username !== undefined, + password: passwordSupplied, + description: opts.description !== undefined, + instruction: opts.instruction !== undefined, }; - const presentFields = Object.entries(mutableFields).filter(([, v]) => v !== undefined); - if (presentFields.length === 0) { + const presentFieldNames = Object.entries(mutableFields) + .filter(([, present]) => present) + .map(([field]) => field); + if (presentFieldNames.length === 0) { throw localValidationError( 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', ); @@ -308,19 +305,36 @@ export async function runUpdate( } const sample: CliUpdateProjectResponse = { id: opts.projectId, - updatedFields: presentFields.map(([k]) => k), + updatedFields: presentFieldNames, updatedAt: '2026-05-16T00:00:00.000Z', }; out.print(sample, data => renderUpdateText(data as CliUpdateProjectResponse)); return sample; } + // Resolve password only on the real path. Dry-run must not touch the + // filesystem, even when --password-file is present. + let password = opts.password; + if (password === undefined && opts.passwordFile !== undefined) { + password = readFileSync(opts.passwordFile, 'utf8').trim(); + } + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-update-${randomUUID()}`; if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderr(`idempotency-key: ${idempotencyKey}`); } - const body = Object.fromEntries(presentFields) as Record; + const bodyFields: Record = { + name: opts.name, + targetUrl: opts.targetUrl, + username: opts.username, + password, + description: opts.description, + instruction: opts.instruction, + }; + const body = Object.fromEntries( + Object.entries(bodyFields).filter(([, v]) => v !== undefined), + ) as Record; const client = makeClient(opts, deps); const updated = await client.patch( `/projects/${encodeURIComponent(opts.projectId)}`, diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 0f05bf7..95d4a35 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -923,6 +923,22 @@ describe('--dry-run subprocess smoke', () => { expect(parsed.id).toBeTruthy(); }, 30_000); + it('project update --dry-run does not read a missing --password-file', async () => { + const result = await runCli([ + 'project', + 'update', + 'proj_anything', + '--password-file', + '/tmp/definitely-not-here-testsprite', + '--dry-run', + '--output', + 'json', + ]); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout) as { updatedFields: string[] }; + expect(parsed.updatedFields).toContain('password'); + }, 30_000); + it('test list --dry-run returns canned TestList', async () => { const result = await runCli([ 'test', From e53257d00887b636a05f0aad09828c13eb28e83c Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:48:31 +0100 Subject: [PATCH 021/117] fix(cli): validate --request-timeout flag and de-duplicate its parser (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `parseRequestTimeoutFlag` was copy-pasted byte-for-byte into five command files (auth, project, usage, init, test). Every copy silently returned `undefined` for an invalid value, so an explicit `--request-timeout 30s` (a natural "30 seconds" typo) resolved to undefined and the command ran with the default 120s deadline — the operator believed they had set a timeout but had not, with no signal. Hoist a single definition into client-factory.ts (next to resolveRequestTimeoutMs and the REQUEST_TIMEOUT_* constants) and make the flag strict: a non-numeric, zero, or negative value now throws a typed VALIDATION_ERROR (exit 5), consistent with every other validated flag (--page-size, --output, --type). Positive out-of-range values are still accepted and clamped by resolveRequestTimeoutMs, and the TESTSPRITE_REQUEST_TIMEOUT_MS env-var path stays lenient by design (a stray global env var should not hard-fail every command). Adds unit coverage for parseRequestTimeoutFlag and a subprocess regression that `--request-timeout 30s` exits 5 instead of falling back to 120s. --- src/commands/auth.ts | 14 +----------- src/commands/init.ts | 12 ++++------ src/commands/project.ts | 13 +---------- src/commands/test.ts | 13 +---------- src/commands/usage.ts | 8 +------ src/lib/client-factory.test.ts | 40 ++++++++++++++++++++++++++++++++++ src/lib/client-factory.ts | 37 +++++++++++++++++++++++++++++++ test/cli.subprocess.test.ts | 18 +++++++++++++++ 8 files changed, 103 insertions(+), 52 deletions(-) diff --git a/src/commands/auth.ts b/src/commands/auth.ts index bd6dc13..5c1b482 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -2,6 +2,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import type { ErrorCode } from '../lib/errors.js'; @@ -337,19 +338,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values — the - * factory applies the same clamp so there is no double-clamp risk. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - function makeOutput(mode: OutputMode, deps: AuthDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } diff --git a/src/commands/init.ts b/src/commands/init.ts index 7615a44..e01215a 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -11,7 +11,10 @@ */ import { Command } from 'commander'; -import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; +import { + parseRequestTimeoutFlag, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; @@ -433,13 +436,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); -} - const SETUP_DESCRIPTION = 'Set up TestSprite: configure your API key and install the TestSprite agent skills for your coding agent'; diff --git a/src/commands/project.ts b/src/commands/project.ts index fd34b47..2af8664 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -4,6 +4,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { ApiError } from '../lib/errors.js'; @@ -531,18 +532,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - function makeClient(opts: CommonOptions, deps: ProjectDeps): HttpClient { return makeHttpClient(opts, { env: deps.env, diff --git a/src/commands/test.ts b/src/commands/test.ts index 8489a01..c6e6c31 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -6,6 +6,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { @@ -7838,18 +7839,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - /** D4: headroom added on top of `--timeout` when deriving the per-request window under `--wait`. */ const WAIT_REQUEST_TIMEOUT_CUSHION_MS = 5_000; diff --git a/src/commands/usage.ts b/src/commands/usage.ts index a539261..d479cab 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -17,6 +17,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { loadConfig } from '../lib/config.js'; @@ -225,13 +226,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); -} - function makeOutput(mode: OutputMode, deps: UsageDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index f9fdc44..61ffe4b 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -5,6 +5,7 @@ import { assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, resetDryRunBannerForTesting, resolveRequestTimeoutMs, } from './client-factory.js'; @@ -213,6 +214,45 @@ describe('resolveRequestTimeoutMs', () => { }); }); +// --------------------------------------------------------------------------- +// parseRequestTimeoutFlag — strict flag parsing (seconds → ms) +// --------------------------------------------------------------------------- + +describe('parseRequestTimeoutFlag', () => { + it('returns undefined when the flag is omitted (factory falls back to env/default)', () => { + expect(parseRequestTimeoutFlag(undefined)).toBeUndefined(); + }); + + it('converts a positive number of seconds to milliseconds', () => { + expect(parseRequestTimeoutFlag('30')).toBe(30_000); + expect(parseRequestTimeoutFlag('1')).toBe(1_000); + expect(parseRequestTimeoutFlag('2.5')).toBe(2_500); + }); + + it('does NOT reject positive out-of-range values — resolveRequestTimeoutMs clamps them', () => { + // 700s is above the 600s cap, but parsing succeeds; the clamp lives in + // resolveRequestTimeoutMs so a large script-supplied value still works. + expect(parseRequestTimeoutFlag('700')).toBe(700_000); + }); + + it.each(['abc', '30s', '0', '-5', 'NaN', 'Infinity', ''])( + 'throws a VALIDATION_ERROR (exit 5) on the invalid flag value %j', + bad => { + let caught: unknown; + try { + parseRequestTimeoutFlag(bad); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('request-timeout'); + }, + ); +}); + // --------------------------------------------------------------------------- // makeHttpClient — requestTimeoutMs propagation // --------------------------------------------------------------------------- diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index fa1f8dc..8f0cc9b 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -180,6 +180,43 @@ export function assertValidEndpointUrl(rawUrl: string): void { } } +/** + * Parse the `--request-timeout ` flag value into milliseconds. + * + * Returns `undefined` when the flag was omitted (the factory then falls back to + * the `TESTSPRITE_REQUEST_TIMEOUT_MS` env var, else the 120s default). + * + * A supplied-but-invalid value (non-numeric, zero, or negative) throws a typed + * VALIDATION_ERROR (exit 5) rather than being silently dropped. An explicit + * `--request-timeout 30s` typo previously resolved to `undefined` and the + * command ran with the default 120s deadline — the operator believed they had + * set a timeout but had not, with no signal. Failing loudly here is consistent + * with every other validated flag (`--page-size`, `--output`, `--type`). + * + * Out-of-range but positive values are intentionally NOT rejected — they flow + * through to {@link resolveRequestTimeoutMs}, which clamps to + * `[REQUEST_TIMEOUT_MIN_MS, REQUEST_TIMEOUT_MAX_MS]`. The env-var path stays + * lenient by design (a stray global env var should not hard-fail every + * command); only the explicit per-invocation flag is strict. + * + * This single definition replaces five byte-identical copies that previously + * lived in `auth`, `project`, `usage`, `init`, and `test` — drift between them + * would have silently changed timeout behaviour depending on the command. + */ +export function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) { + // Surface the offending value in the message (same as assertValidEndpointUrl) + // so the operator sees exactly what they typed. + throw localValidationError( + 'request-timeout', + `"${raw}" is not valid — must be a positive number of seconds`, + ); + } + return Math.round(n * 1000); // seconds → milliseconds +} + export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {}): HttpClient { const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const env = deps.env ?? process.env; diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 95d4a35..b0537b1 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -499,6 +499,24 @@ describe('project list subprocess', () => { const parsed = JSON.parse(result.stderr) as { error: { code: string } }; expect(parsed.error.code).toBe('VALIDATION_ERROR'); }, 30_000); + + it('--request-timeout 30s exits 5 (VALIDATION_ERROR), not a silent fallback to 120s', async () => { + // Previously an invalid flag value resolved to `undefined` and the command + // silently ran with the default 120s deadline — the operator believed they + // had set a timeout but had not. Now the explicit flag is validated like + // every other flag. + const result = await runCli( + ['--output', 'json', '--request-timeout', '30s', 'project', 'list'], + { + TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_URL: baseUrl, + }, + ); + expect(result.exitCode).toBe(5); + const parsed = JSON.parse(result.stderr) as { error: { code: string; nextAction: string } }; + expect(parsed.error.code).toBe('VALIDATION_ERROR'); + expect(parsed.error.nextAction).toContain('request-timeout'); + }, 30_000); }); describe('malformed --endpoint-url is rejected (exit 5), not retried as a network error', () => { From cff19aed2e3cd1cc2242c036f0410bcf906f025e Mon Sep 17 00:00:00 2001 From: Contributor Date: Fri, 3 Jul 2026 04:23:57 +0200 Subject: [PATCH 022/117] fix(rerun): emit partial stdout on TimeoutError in single-FE rerun --wait Apply the fix in src/commands/test.ts. When the overall --timeout polling deadline is exceeded on a single FE rerun, emit {runId, status:"running"} to stdout before exit 7. Co-authored-by: Cursor --- src/commands/test.rerun.spec.ts | 75 +++++++++++++++++++++++++++++++++ src/commands/test.ts | 12 ++++++ 2 files changed, 87 insertions(+) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 86c8e89..3645c12 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4630,3 +4630,78 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); + +// --------------------------------------------------------------------------- +// TimeoutError on single FE rerun --wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + let fetchCallCount = 0; + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + fetchCallCount++; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + const runningRun: RunResponse = { + ...makeTerminalRun(rerunResp.runId, 'passed'), + status: 'running', + finishedAt: null, + }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 0, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch(e => e); + + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(parsed.runId).toBe(rerunResp.runId); + expect(parsed.status).toBe('running'); + + void fetchCallCount; + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index 08e4fe2..69239ca 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -6099,6 +6099,18 @@ export async function runTestRerun( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${rerunResp.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(timeoutPartial, data => { + const p = data as typeof timeoutPartial; + return [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', From 6e7f2b3f0aafa61328c01daf276497b3030b192a Mon Sep 17 00:00:00 2001 From: Resque Date: Sun, 5 Jul 2026 23:27:55 +0400 Subject: [PATCH 023/117] fix(test): reject whitespace-only --name in test update (parity with test create) (#39) * fix(test): reject whitespace-only --name in test update (parity with test create) * style: run prettier on src/commands/test.ts --- src/commands/test.test.ts | 25 +++++++++++++++++++++++++ src/commands/test.ts | 6 ++++++ 2 files changed, 31 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 49127bd..f5ec62a 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -5777,6 +5777,31 @@ describe('runUpdate', () => { expect(called).toBe(0); }); + it('rejects a whitespace-only --name before sending (parity with test create)', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: ' ', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'name' }), + }); + expect(called).toBe(0); + }); + it('renders text mode with one line per updated field', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); diff --git a/src/commands/test.ts b/src/commands/test.ts index c6e6c31..d001c6c 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1243,6 +1243,12 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); requireNonEmpty('test-id', opts.testId); // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError( + 'name', + 'must be a non-empty string (whitespace-only is not allowed)', + ); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('name', 'must be at most 200 characters'); } From dc4d3375dc6ff0cff0a9104bbc1776898f14ed5d Mon Sep 17 00:00:00 2001 From: Rahul Joshi <186129212+crypticsaiyan@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:58:51 +0530 Subject: [PATCH 024/117] fix(test): guard parseDuration --since against overflow with VALIDATION_ERROR (#27) --- src/commands/test.result.history.spec.ts | 18 ++++++++++++++++++ src/commands/test.ts | 12 ++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index 6f6cdd2..a2d2018 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -169,6 +169,24 @@ describe('parseDuration', () => { it('case-insensitive day suffix', () => { expect(parseDuration('7D', NOW)).toBe('2026-05-27T12:00:00.000Z'); }); + + it('overflow hours throws VALIDATION_ERROR instead of crashing', () => { + expect(() => parseDuration('99999999999h', NOW)).toThrow(); + try { + parseDuration('99999999999h', NOW); + } catch (err: unknown) { + expect((err as { code?: string }).code).toBe('VALIDATION_ERROR'); + } + }); + + it('overflow days throws VALIDATION_ERROR instead of crashing', () => { + expect(() => parseDuration('99999999999d', NOW)).toThrow(); + try { + parseDuration('99999999999d', NOW); + } catch (err: unknown) { + expect((err as { code?: string }).code).toBe('VALIDATION_ERROR'); + } + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/test.ts b/src/commands/test.ts index d001c6c..07e4e29 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3809,12 +3809,20 @@ export function parseDuration(raw: string, now: Date = new Date()): string { const hourMatch = /^(\d+)h$/i.exec(raw); if (hourMatch) { const hours = Number(hourMatch[1]); - return new Date(now.getTime() - hours * 60 * 60 * 1000).toISOString(); + const result = new Date(now.getTime() - hours * 60 * 60 * 1000); + if (!Number.isFinite(result.getTime())) { + throw localValidationError('since', 'duration is too large; maximum is ~1141552511h'); + } + return result.toISOString(); } const dayMatch = /^(\d+)d$/i.exec(raw); if (dayMatch) { const days = Number(dayMatch[1]); - return new Date(now.getTime() - days * 24 * 60 * 60 * 1000).toISOString(); + const result = new Date(now.getTime() - days * 24 * 60 * 60 * 1000); + if (!Number.isFinite(result.getTime())) { + throw localValidationError('since', 'duration is too large; maximum is ~47564688d'); + } + return result.toISOString(); } // Pass-through: ISO timestamp or epoch value — server validates. return raw; From 6b90ff405006ff0731c944bec961ae7d120c2250 Mon Sep 17 00:00:00 2001 From: Resque Date: Sun, 5 Jul 2026 23:29:07 +0400 Subject: [PATCH 025/117] feat(cli): respect NO_COLOR environment variable per no-color.org (#12) * feat(cli): respect NO_COLOR environment variable per no-color.org * fix(ticker): treat empty NO_COLOR as color-enabled per no-color.org --- DOCUMENTATION.md | 13 +++++---- src/lib/ticker.spec.ts | 65 +++++++++++++++++++++++++++++++++++++++++- src/lib/ticker.ts | 35 +++++++++++++++++++++++ 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 700671b..461fc63 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -426,12 +426,13 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ------------------------------- | --------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| Variable | Purpose | +| ------------------------------- | --------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | ### Scopes diff --git a/src/lib/ticker.spec.ts b/src/lib/ticker.spec.ts index d449900..72760a9 100644 --- a/src/lib/ticker.spec.ts +++ b/src/lib/ticker.spec.ts @@ -3,7 +3,7 @@ */ import { describe, expect, it, vi } from 'vitest'; -import { createTicker } from './ticker.js'; +import { createTicker, isNoColor } from './ticker.js'; describe('createTicker — non-TTY (CI mode)', () => { it('update is a no-op (no writes)', () => { @@ -222,3 +222,66 @@ describe('createTicker — spy on process.stderr', () => { } }); }); + +describe('createTicker — NO_COLOR support', () => { + it('suppresses ANSI escape sequences when noColor=true on TTY', () => { + const lines: string[] = []; + const raw: string[] = []; + const ticker = createTicker( + line => lines.push(line), + true, // isTTY = true + text => raw.push(text), + true, // noColor = true + ); + ticker.update('progress'); + // Should use stderrWrite (line-oriented) instead of rawWrite with ANSI + expect(raw).toHaveLength(0); + expect(lines).toHaveLength(1); + expect(lines[0]!).not.toContain('\x1b[2K'); + expect(lines[0]!).not.toContain('\r'); + expect(lines[0]!).toContain('progress'); + }); + + it('finalize emits plain text without ANSI when noColor=true on TTY', () => { + const lines: string[] = []; + const raw: string[] = []; + const ticker = createTicker( + line => lines.push(line), + true, + text => raw.push(text), + true, // noColor = true + ); + ticker.finalize('done'); + expect(raw).toHaveLength(0); + expect(lines).toHaveLength(1); + expect(lines[0]!).not.toContain('\x1b[2K'); + expect(lines[0]!).toContain('done'); + }); + + it('normal ANSI output when noColor=false on TTY', () => { + const raw: string[] = []; + const ticker = createTicker( + () => {}, + true, + text => raw.push(text), + false, // noColor = false + ); + ticker.update('progress'); + expect(raw).toHaveLength(1); + expect(raw[0]!).toContain('\x1b[2K\r'); + }); +}); + +describe('isNoColor', () => { + it('returns true when NO_COLOR is set to a non-empty value', () => { + expect(isNoColor({ NO_COLOR: '1' })).toBe(true); + expect(isNoColor({ NO_COLOR: 'true' })).toBe(true); + }); + + it('returns false when NO_COLOR is absent or empty', () => { + expect(isNoColor({})).toBe(false); + expect(isNoColor({ OTHER_VAR: '1' })).toBe(false); + // Per https://no-color.org/, an empty NO_COLOR does NOT disable color. + expect(isNoColor({ NO_COLOR: '' })).toBe(false); + }); +}); diff --git a/src/lib/ticker.ts b/src/lib/ticker.ts index 4cae6b1..d9eaed2 100644 --- a/src/lib/ticker.ts +++ b/src/lib/ticker.ts @@ -8,6 +8,9 @@ * - Uses `\r` + ANSI clear-line to overwrite in place on TTY * - On terminal, emits one final line + newline then prints the result * - `--output json` disables the ticker (caller doesn't create one) + * - Respects the NO_COLOR env var (https://no-color.org/): when set, + * ANSI escape sequences are suppressed and updates are emitted as + * plain lines instead of in-place overwrites. * * Overhead: <2ms per update (no syscalls beyond a single write). * @@ -25,6 +28,15 @@ export interface Ticker { finalize(line?: string): void; } +/** + * Returns true when NO_COLOR is present in the environment and is not + * an empty string, per https://no-color.org/. + */ +export function isNoColor(env: NodeJS.ProcessEnv = process.env): boolean { + const value = env.NO_COLOR; + return typeof value === 'string' && value.length > 0; +} + /** * Create a ticker bound to the given stderr writer. Respects * `isTTY` to silently no-op in CI environments. @@ -35,11 +47,14 @@ export interface Ticker { * @param stderrRaw - optional raw writer (no \n appended); used for * the carriage-return + clear-line trick. Defaults to * `process.stderr.write.bind(process.stderr)`. + * @param noColor - whether to suppress ANSI escape sequences. + * Defaults to checking `NO_COLOR` env var per https://no-color.org/. */ export function createTicker( stderrWrite: (line: string) => void, isTTY?: boolean, stderrRaw?: (text: string) => void, + noColor?: boolean, ): Ticker { const tty = isTTY ?? (typeof process !== 'undefined' ? process.stderr.isTTY === true : false); const rawWrite = @@ -47,6 +62,7 @@ export function createTicker( (typeof process !== 'undefined' ? (text: string) => process.stderr.write(text) : (_text: string) => undefined); + const suppressAnsi = noColor ?? isNoColor(); let lastLength = 0; @@ -58,6 +74,25 @@ export function createTicker( }; } + if (suppressAnsi) { + // TTY but NO_COLOR: emit plain-text lines without ANSI escape sequences. + return { + update(line: string): void { + const stamped = `${new Date().toISOString()} ${line}`; + stderrWrite(stamped); + lastLength = stamped.length; + }, + finalize(line?: string): void { + if (line !== undefined) { + const stamped = `${new Date().toISOString()} ${line}`; + stderrWrite(stamped); + lastLength = stamped.length; + } + void stderrWrite; + }, + }; + } + return { update(line: string): void { // ANSI ESC[2K clears the entire line; \r moves to column 0. From 205b1fb37f2778bf7a3ddaf7c1ab458dec9ae119 Mon Sep 17 00:00:00 2001 From: Aldo Rizona Date: Mon, 6 Jul 2026 02:29:22 +0700 Subject: [PATCH 026/117] fix(paginate): bound cursor loops without dropping empty pages (#48) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(paginate): break loop on empty items with non-null nextToken paginate() in pagination.ts entered an infinite loop when the API returned { items: [], nextToken: 'cursor' }. The loop only checked nextToken !== null, never whether items was empty. Any filtered list command returning zero results would hang indefinitely. Add an early-exit guard: if the page contains zero items, break regardless of nextToken. Includes 5 unit tests covering the bug reproduction, maxItems cap, single-page, and consecutive empty pages. Closes #35 Signed-off-by: Aldo Rizona * style(test): reflow pagination.test.ts to satisfy prettier format:check gate Pure mechanical prettier reflow of two vi.fn(async () => ({...})) callbacks in test/pagination.test.ts. No logic change — only line wrapping to satisfy the format:check CI gate (npm run format:check). All other gates already pass. Coverage remains >=80% on all 4 metrics (lines/statements/functions/branches). * fix(paginate): bound cursor loops without dropping empty pages --------- Signed-off-by: Aldo Rizona --- src/lib/pagination.test.ts | 59 +++++++++++++++++++++++++++++++++++++- src/lib/pagination.ts | 31 +++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/lib/pagination.test.ts b/src/lib/pagination.test.ts index a11ea33..8a20295 100644 --- a/src/lib/pagination.test.ts +++ b/src/lib/pagination.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest'; import { ApiError } from './errors.js'; -import { paginate, validatePaginationFlags, type FetchPage, type Page } from './pagination.js'; +import { + MAX_AUTO_PAGES, + paginate, + validatePaginationFlags, + type FetchPage, + type Page, +} from './pagination.js'; function makePages(pages: Page[]): { fetchPage: FetchPage; @@ -121,4 +127,55 @@ describe('paginate', () => { await expect(paginate(fetchPage, { pageSize: 0 })).rejects.toBeInstanceOf(ApiError); expect(calls).toHaveLength(0); }); + + it('continues through empty cursor pages until later data', async () => { + const { fetchPage, calls } = makePages([ + { items: [], nextToken: 'cursor-1' }, + { items: [1], nextToken: null }, + ]); + + const page = await paginate(fetchPage); + + expect(page.items).toEqual([1]); + expect(page.nextToken).toBeNull(); + expect(calls).toHaveLength(2); + expect(calls[1]!.cursor).toBe('cursor-1'); + }); + + it('rejects when API repeats a non-null cursor without making progress', async () => { + const { fetchPage, calls } = makePages([{ items: [], nextToken: 'cursor-1' }]); + + await expect(paginate(fetchPage)).rejects.toMatchObject({ + code: 'UNAVAILABLE', + details: expect.objectContaining({ reason: 'repeated_next_token' }), + }); + + expect(calls).toHaveLength(2); + }); + + it('rejects when auto-pagination exceeds the page safety cap', async () => { + const calls: Array<{ pageSize: number; cursor: string | undefined }> = []; + const fetchPage: FetchPage = async args => { + calls.push(args); + return { + items: [], + nextToken: + args.cursor === undefined ? 'cursor-1' : `cursor-${Number(args.cursor.slice(7)) + 1}`, + }; + }; + + let thrown: unknown; + try { + await paginate(fetchPage); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(ApiError); + expect(thrown).toMatchObject({ + code: 'UNAVAILABLE', + details: expect.objectContaining({ reason: 'max_pages_exceeded' }), + }); + expect(calls).toHaveLength(MAX_AUTO_PAGES); + }); }); diff --git a/src/lib/pagination.ts b/src/lib/pagination.ts index ba63f9e..cae7581 100644 --- a/src/lib/pagination.ts +++ b/src/lib/pagination.ts @@ -1,5 +1,5 @@ import type { HttpClient } from './http.js'; -import { localValidationError } from './errors.js'; +import { ApiError, localValidationError } from './errors.js'; /** * Page shape returned by every list endpoint per @@ -22,6 +22,7 @@ export interface PaginationFlags { const HARD_PAGE_SIZE_CAP = 100; const DEFAULT_PAGE_SIZE = 25; +export const MAX_AUTO_PAGES = 1000; /** * Validates and normalizes pagination flags. Per the CLI OpenAPI spec @@ -91,14 +92,24 @@ export async function paginate( const items: T[] = []; let cursor: string | undefined = flags.startingToken; let lastNextToken: string | null = null; + let pagesFetched = 0; + const seenNextTokens = new Set(); + if (cursor !== undefined) seenNextTokens.add(cursor); while (true) { const remaining = maxItems !== undefined ? maxItems - items.length : Infinity; if (remaining <= 0) break; + if (pagesFetched >= MAX_AUTO_PAGES) { + throw paginationSafetyError('max_pages_exceeded', { + maxPages: MAX_AUTO_PAGES, + lastCursor: cursor ?? null, + }); + } const callPageSize = Number.isFinite(remaining) ? Math.min(pageSize, remaining) : pageSize; const page = await fetchPage({ pageSize: callPageSize, cursor }); + pagesFetched += 1; lastNextToken = page.nextToken; for (const item of page.items) { @@ -107,12 +118,30 @@ export async function paginate( } if (page.nextToken === null) break; + if (seenNextTokens.has(page.nextToken)) { + throw paginationSafetyError('repeated_next_token', { + cursor: page.nextToken, + pagesFetched, + }); + } + seenNextTokens.add(page.nextToken); cursor = page.nextToken; } return { items, nextToken: lastNextToken }; } +function paginationSafetyError(reason: string, details: Record): ApiError { + return ApiError.fromEnvelope({ + code: 'UNAVAILABLE', + message: 'Pagination did not make progress safely.', + nextAction: + 'Retry later. If the problem continues, contact TestSprite support with the cursor details.', + requestId: 'local', + details: { reason, ...details }, + }); +} + /** * Drop-in helper for commands that take a single page and surface * the cursor verbatim (no auto-follow). Used when the caller passed From 06d2c4deb5c2e51123ca6358673af782f2fb56a6 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:29:38 -0500 Subject: [PATCH 027/117] fix(project): validate list flags before client setup (#150) --- src/commands/project.test.ts | 41 ++++++++++++++++++++++++++++++++++++ src/commands/project.ts | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 1fcbd99..268df16 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -199,6 +199,47 @@ describe('runList', () => { }); }); + it('rejects invalid pagination before requiring credentials', async () => { + const credentialsPath = join(mkdtempSync(join(tmpdir(), 'cli-p2-no-creds-')), 'credentials'); + const fetchImpl = vi.fn(); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, pageSize: 1.5 }, + { credentialsPath, fetchImpl: fetchImpl as unknown as typeof globalThis.fetch }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('rejects invalid dry-run pagination before emitting the dry-run banner', async () => { + const stderr: string[] = []; + const fetchImpl = vi.fn(); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, dryRun: true, pageSize: 1.5 }, + { + credentialsPath: join(mkdtempSync(join(tmpdir(), 'cli-p2-dryrun-')), 'credentials'), + fetchImpl: fetchImpl as unknown as typeof globalThis.fetch, + stderr: line => stderr.push(line), + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(stderr.join('\n')).not.toContain(DRY_RUN_BANNER); + }); + it('rejects pageSize=101 with VALIDATION_ERROR exit 5 (Fix 7 — upper-bound enforced client-side)', async () => { // Previously silently clamped to 100; now rejected so callers get fast feedback. const { credentialsPath } = makeCreds(); diff --git a/src/commands/project.ts b/src/commands/project.ts index 2af8664..2510672 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -51,13 +51,13 @@ export async function runList( deps: ProjectDeps = {}, ): Promise> { const out = makeOutput(opts.output, deps); - const client = makeClient(opts, deps); const paginationFlags: PaginationFlags = validatePaginationFlags({ pageSize: opts.pageSize, startingToken: opts.startingToken, maxItems: opts.maxItems, }); + const client = makeClient(opts, deps); // When the user explicitly passed a page-size flag and did NOT ask // for --max-items, treat that as a "give me one page and the cursor" From 495db2f9f098b30cbb9db3bd7fe809332150cf7b Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:29:54 -0500 Subject: [PATCH 028/117] fix(test): validate list status before client setup (#152) --- src/commands/test.test.ts | 28 ++++++++++++++++++++++++++++ src/commands/test.ts | 12 ++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index f5ec62a..dc0527e 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -475,6 +475,34 @@ describe('runList', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', details: { field: 'page-size' } }); }); + it('rejects invalid --status before requiring credentials', async () => { + const credentialsPath = join(mkdtempSync(join(tmpdir(), 'cli-list-status-')), 'credentials'); + const fetchImpl = vi.fn(); + + await expect( + runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + status: 'notastatus', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => undefined, + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'status' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('forwards a server-side VALIDATION_ERROR envelope as ApiError exit 5', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ diff --git a/src/commands/test.ts b/src/commands/test.ts index 07e4e29..1644f85 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -452,6 +452,12 @@ export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise

= { projectId: opts.projectId, type: opts.type, From 25ed285cbadb7c1a7a56cac1fd2ec25f8cdb437f Mon Sep 17 00:00:00 2001 From: nopp Date: Mon, 6 Jul 2026 02:30:10 +0700 Subject: [PATCH 029/117] fix: reject blank project passwords (#139) --- src/commands/project.test.ts | 51 ++++++++++++++++++++++++++++++++++++ src/commands/project.ts | 6 +++++ 2 files changed, 57 insertions(+) diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 268df16..f63928c 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -637,6 +637,33 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + it('rejects a whitespace-only --password with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network - validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Password Guard Project', + targetUrl: 'https://example.com', + password: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); }); // --------------------------------------------------------------------------- @@ -739,6 +766,30 @@ describe('runUpdate', () => { expect(fetchImpl).not.toHaveBeenCalled(); }); + it('rejects a whitespace-only --password with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + password: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); it('P7 — dry-run returns canned shape without network call', async () => { resetDryRunBannerForTesting(); const { credentialsPath } = makeCreds(); diff --git a/src/commands/project.ts b/src/commands/project.ts index 2510672..ef4d5e8 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -150,6 +150,9 @@ export async function runCreate( if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } + if (opts.password !== undefined && opts.password.trim().length === 0) { + throw localValidationError('--password must not be empty or whitespace-only'); + } // P1-3: client-side length checks matching server limits. if (opts.name !== undefined && opts.name.length > 200) { @@ -264,6 +267,9 @@ export async function runUpdate( if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } + if (opts.password !== undefined && opts.password.trim().length === 0) { + throw localValidationError('--password must not be empty or whitespace-only'); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } From 87a6dff24e5f9744a1c5f6c9335100e60961e000 Mon Sep 17 00:00:00 2001 From: nopp Date: Mon, 6 Jul 2026 02:30:27 +0700 Subject: [PATCH 030/117] fix: reject directory code output paths (#140) --- src/commands/test.test.ts | 25 +++++++++++++++++++++++++ src/commands/test.ts | 11 +++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index dc0527e..d3b45ca 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -1550,6 +1550,31 @@ describe('runCodeGet', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('--out rejects an existing directory path with VALIDATION_ERROR (exit 5) before any network I/O', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-dir-')); + let fetchCalls = 0; + const fetchImpl = (() => { + fetchCalls += 1; + return Promise.resolve(new Response('{}')); + }) as typeof globalThis.fetch; + + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: dir, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchCalls).toBe(0); + expect(readdirSync(dir)).toEqual([]); + }); + // Regression: a parent dir that doesn't exist used to surface as exit 1 // (TRANSPORT_ERROR) — `createWriteStream` opens lazily and ENOENT fires // mid-write. Synchronous parent stat keeps every `--out` user-input diff --git a/src/commands/test.ts b/src/commands/test.ts index 1644f85..5d6eb80 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -7953,6 +7953,17 @@ function openOutputFile(rawPath: string): FileSink { if (!parentStat.isDirectory()) { throw localValidationError('out', `parent path is not a directory: ${parent}`); } + let targetStat; + try { + targetStat = statSync(resolved); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + throw localValidationError('out', `cannot stat output path: ${resolved}`); + } + } + if (targetStat?.isDirectory()) { + throw localValidationError('out', `must point to a file, not a directory: ${resolved}`); + } const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); const sink: FileSink = { stream, path: resolved, tmpPath, error: null }; From b5879de422891208a9b5a57fec8d45d04267c587 Mon Sep 17 00:00:00 2001 From: nopp Date: Mon, 6 Jul 2026 02:30:42 +0700 Subject: [PATCH 031/117] fix: reject malformed api keys (#141) --- src/lib/client-factory.test.ts | 55 +++++++++++++++++++++++++++++++++- src/lib/client-factory.ts | 17 +++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index 61ffe4b..ab5fd99 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -1,7 +1,8 @@ -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { DRY_RUN_API_KEY, DRY_RUN_BANNER, + assertValidApiKeyHeaderValue, assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, @@ -331,6 +332,58 @@ describe('makeHttpClient — real path (regression)', () => { // assertValidEndpointUrl — endpoint syntax guard (NOT an SSRF guard) // --------------------------------------------------------------------------- +describe('makeHttpClient - API key validation', () => { + it.each([ + ['newline', 'sk-user-abc\ndef'], + ['carriage return', 'sk-user-abc\rdef'], + ['smart dash', 'sk-user-abc\u2013def'], + ['smart quote', 'sk-user-\u201cabc\u201d'], + ['emoji', 'sk-user-abc\u{1f600}'], + ['whitespace-only', ' '], + ])('rejects a malformed configured API key with %s before fetch/retry', (_label, apiKey) => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: apiKey } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }); +}); + +describe('assertValidApiKeyHeaderValue', () => { + it('accepts a normal ASCII API key value', () => { + expect(() => assertValidApiKeyHeaderValue('sk-user-abc-def_123')).not.toThrow(); + }); + + it('throws a VALIDATION_ERROR (exit 5) for a key that cannot be sent as x-api-key', () => { + let caught: unknown; + try { + assertValidApiKeyHeaderValue('sk-user-abc\u2013def'); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }); +}); + describe('assertValidEndpointUrl', () => { it('accepts http(s) URLs, including private / localhost hosts (self-hosted, dev, mock)', () => { for (const url of [ diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index 8f0cc9b..829410d 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -180,6 +180,22 @@ export function assertValidEndpointUrl(rawUrl: string): void { } } +export function assertValidApiKeyHeaderValue(apiKey: string): void { + const reason = + 'must be a non-empty HTTP header value; paste the raw key without smart punctuation, emoji, or line breaks'; + + if (apiKey.trim().length === 0) { + throw localValidationError('api-key', reason, undefined, 'field'); + } + + for (let i = 0; i < apiKey.length; i += 1) { + const code = apiKey.charCodeAt(i); + if (code < 0x20 || code === 0x7f || code > 0xff) { + throw localValidationError('api-key', reason, undefined, 'field'); + } + } +} + /** * Parse the `--request-timeout ` flag value into milliseconds. * @@ -251,6 +267,7 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} // VALIDATION_ERROR rather than an opaque URL throw or a retried "fetch failed". assertValidEndpointUrl(config.apiUrl); if (!config.apiKey) throw ApiError.authRequired(); + assertValidApiKeyHeaderValue(config.apiKey); return new HttpClient({ baseUrl: facadeBaseUrl(config.apiUrl), apiKey: config.apiKey, From c0b52f852394e0199e663ddf02262ad2feb9fe51 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:30:59 -0500 Subject: [PATCH 032/117] fix(setup): validate endpoint before key check (#149) --- src/commands/auth.test.ts | 94 +++++++++++++++++++++++++++++++++++++++ src/commands/auth.ts | 5 ++- src/commands/init.test.ts | 30 +++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 5207085..8a7ae8b 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -164,6 +164,100 @@ describe('runConfigure', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('rejects a malformed endpoint before key validation fetch', async () => { + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: true, + endpointUrl: 'not-a-url', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(capture.stderr.join('\n')).not.toContain('API key rejected'); + }); + + it('rejects a non-http endpoint before key validation fetch', async () => { + const { deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: true, + endpointUrl: 'ftp://example.com', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + + it('rejects a malformed dry-run endpoint before emitting dry-run output', async () => { + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: false, + dryRun: true, + endpointUrl: 'not-a-url', + }, + { + ...deps, + env: {}, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(capture.stderr.join('\n')).not.toContain('[dry-run]'); + expect(capture.stdout).toEqual([]); + }); + it('prompts only for the API key (never the endpoint) and defaults to prod', async () => { const { capture, deps } = makeCapture(); // Prompt object exposes ONLY `secret`. If runConfigure tried to prompt for diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 5c1b482..1ec3e40 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -1,5 +1,6 @@ import { Command } from 'commander'; import { + assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, parseRequestTimeoutFlag, @@ -87,8 +88,9 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // Print the canned success shape so an agent sees exactly the JSON it // would get on a real configure (modulo the endpoint string). if (opts.dryRun) { - emitDryRunBanner(stderr); const apiUrl = opts.endpointUrl ?? envApiUrl ?? DEFAULT_API_URL; + assertValidEndpointUrl(apiUrl); + emitDryRunBanner(stderr); stderr(`[dry-run] would write credentials for profile="${opts.profile}" to ${credentialsPath}`); out.print({ profile: opts.profile, apiUrl, status: 'configured' }, data => { const d = data as { profile: string; apiUrl: string }; @@ -114,6 +116,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // api_url doesn't silently validate a new key against the default endpoint. const resolvedFromProfile = existingProfile?.apiUrl; const apiUrl = opts.endpointUrl ?? envApiUrl ?? resolvedFromProfile ?? DEFAULT_API_URL; + assertValidEndpointUrl(apiUrl); if (opts.fromEnv) { apiKey = env.TESTSPRITE_API_KEY?.trim(); diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index be19095..75fdff0 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -542,6 +542,36 @@ describe('runInit — codex-review hardening', () => { expect(fetchImpl).toHaveBeenCalled(); }); + it('rejects malformed --endpoint-url before setup key verification', async () => { + const { captured, deps } = makeCapture(); + const fetchImpl = makeOkFetch(); + + await expect( + runInit( + makeBaseOpts({ + fromEnv: true, + endpointUrl: 'not-a-url', + noAgent: true, + output: 'json', + }), + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + fetchImpl, + credentialsPath, + isTTY: false, + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(captured.stderr.join('\n')).not.toContain('API key rejected'); + }); + it('whoami banner uses --api-key, not a stale TESTSPRITE_API_KEY in env (E2E 2026-06-09)', async () => { const { captured, deps } = makeCapture(); // Key-aware fetch: only the real key gets a 200 + identity; the stale env key 401s. From 0de372c51d1282d61ccf11b1c3c7b1f1a0d22398 Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:31:15 +0100 Subject: [PATCH 033/117] fix(http): map non-JSON 200 responses to a typed error envelope (#166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A successful (200) response whose body is not JSON — a misconfigured endpoint, a proxy / captive-portal / login page returning HTML with a success status, or an empty body — caused the OK path to call raw `response.json()`, whose SyntaxError escaped to the top-level handler. That produced an opaque exit 1 and, under --output json, a bare `{"error":""}` that breaks the typed-envelope contract every other error honors (the non-OK path already reads defensively via safeReadJson). Wrap the OK-path parse failure in a typed INTERNAL ApiError (exit 1 unchanged) that carries the requestId, names the likely cause, and points the operator at their endpoint config; details include the HTTP status, content-type, and underlying parse message. Abort/timeout errors mid-read keep their existing classification. Fixes #94 --- src/lib/http.test.ts | 47 +++++++++++++++++++++++++++++++++++++ src/lib/http.ts | 55 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 3387eac..0320367 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -140,6 +140,53 @@ describe('HttpClient happy path', () => { }); }); +describe('HttpClient — 200 response with a non-JSON body', () => { + function htmlResponse(status = 200): Response { + return new Response('Login', { + status, + headers: { 'content-type': 'text/html' }, + }); + } + + it('throws a typed INTERNAL ApiError (not a raw SyntaxError) with the requestId and details', async () => { + const fetchImpl = vi.fn(async () => htmlResponse()); + const client = makeClient(fetchImpl as unknown as typeof fetch); + let caught: unknown; + try { + await client.get('/me'); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('INTERNAL'); + expect(apiErr.exitCode).toBe(1); + expect(apiErr.requestId).toMatch(/^cli_/); + expect(apiErr.message).toContain('non-JSON response'); + expect(apiErr.nextAction).toContain('TESTSPRITE_API_URL'); + expect(apiErr.details).toMatchObject({ httpStatus: 200, contentType: 'text/html' }); + expect(String(apiErr.details.parseError)).toContain('JSON'); + }); + + it('does not retry a malformed 200 body (single fetch call)', async () => { + const fetchImpl = vi.fn(async () => htmlResponse()); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.get('/me')).rejects.toBeInstanceOf(ApiError); + // A non-JSON success body is a hard config error, not a transient transport + // failure — it must not burn the retry budget. + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('handles an empty 200 body the same way', async () => { + const fetchImpl = vi.fn( + async () => + new Response('', { status: 200, headers: { 'content-type': 'application/json' } }), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.get('/me')).rejects.toMatchObject({ code: 'INTERNAL', exitCode: 1 }); + }); +}); + describe('HttpClient error mapping', () => { it('does not retry AUTH_INVALID and exits 3', async () => { const fetchImpl = vi.fn(async () => errorEnvelopeResponse(401, 'AUTH_INVALID')); diff --git a/src/lib/http.ts b/src/lib/http.ts index 83ca0f1..b5dabb1 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -498,7 +498,13 @@ export class HttpClient { } catch (err) { // A timeout/abort can fire mid-body-read (headers received, stream stalls). this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - throw err; + // Otherwise the successful response body was not valid JSON — a + // misconfigured endpoint, a proxy / captive-portal / login page that + // returns HTML with a 200 status, or an empty body. Surface a typed + // error carrying the requestId instead of letting the raw SyntaxError + // escape to index.ts, where it would print a bare `{"error":"..."}` + // and break the --output json envelope contract. + throw malformedResponseError(response, requestId, err); } } @@ -692,6 +698,53 @@ async function safeReadJson(response: Response): Promise { } } +/** + * Build a typed error for a successful response whose body could not be parsed + * as JSON. + * + * The CLI expects every API response to be a JSON envelope. When a `200 OK` + * carries a non-JSON body — a misconfigured endpoint, a proxy / captive-portal + * / SSO login page returning HTML with a success status, or an empty body — + * `response.json()` throws a raw `SyntaxError`. Left unhandled it escapes to + * the top-level handler in `index.ts`, which prints a bare + * `{"error":""}` under `--output json` (breaking the + * typed-envelope contract every other error honors) and gives the operator no + * actionable context. + * + * This wraps it in a typed `INTERNAL` `ApiError` (exit 1, unchanged) that + * carries the `requestId`, names the likely cause, and points the operator at + * their endpoint configuration. `details` includes the HTTP status, the + * response `content-type` (when present), and the underlying parse message. + */ +export function malformedResponseError( + response: Response, + requestId: string, + cause: unknown, +): ApiError { + const contentType = response.headers.get('content-type') ?? undefined; + const parseError = cause instanceof Error ? cause.message : String(cause); + const contentTypeNote = contentType ? ` (content-type: ${contentType})` : ''; + return new ApiError( + { + code: 'INTERNAL', + message: + `The server returned a non-JSON response${contentTypeNote} for an HTTP ${response.status}. ` + + `This usually means the endpoint is not the TestSprite API — a proxy, captive portal, or ` + + `login page can return HTML with a success status.`, + nextAction: + 'Check that --endpoint-url / TESTSPRITE_API_URL points at the TestSprite API ' + + '(default https://api.testsprite.com), then retry.', + requestId, + details: { + httpStatus: response.status, + ...(contentType ? { contentType } : {}), + parseError, + }, + }, + response.status, + ); +} + export function parseRetryAfter(headerValue: string | null): number | undefined { if (!headerValue) return undefined; const numeric = Number(headerValue); From b675630b852e73b1f474d905a060cbb9eeb40b0a Mon Sep 17 00:00:00 2001 From: Sahil Rakhaiya <144577420+SahilRakhaiya05@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:01:37 +0530 Subject: [PATCH 034/117] feat(test): JUnit XML report export for batch --wait runs (#96) * feat(test): JUnit XML report export for batch --wait runs * fix(ci): prettier formatting and help snapshot alignment for report flags * fix(test): address CodeRabbit review on JUnit report export * fix(test): add projectId to CliBatchRunFreshResult for JUnit inference --- CHANGELOG.md | 4 + DOCUMENTATION.md | 21 ++ src/commands/test.run.spec.ts | 220 +++++++++++- src/commands/test.ts | 141 +++++++- src/lib/dry-run/samples.test.ts | 20 +- src/lib/dry-run/samples.ts | 32 ++ src/lib/junit-report.test.ts | 330 ++++++++++++++++++ src/lib/junit-report.ts | 267 ++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 102 +++--- 9 files changed, 1087 insertions(+), 50 deletions(-) create mode 100644 src/lib/junit-report.test.ts create mode 100644 src/lib/junit-report.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e0d49..a4ab9b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +### Added + +- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. + ## [0.2.0] - 2026-06-29 ### Added diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 461fc63..a1ac967 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -344,8 +344,18 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json + +# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json ``` +Batch `--report` flags apply only to `test run --all --wait` (and batch `test rerun --wait`). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. Optional `--report-suite-name ` overrides the default `testsprite:` suite name. + `--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. #### `testsprite test rerun [test-id...]` @@ -365,10 +375,20 @@ testsprite test rerun test_be_xxxx --skip-dependencies --output json # Rerun every test in a project (batch) testsprite test rerun --all --project proj_xxxxxxxx --wait --max-concurrency 4 --output json +# Batch rerun with JUnit XML for CI +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json + # Several specific tests testsprite test rerun test_aaaa test_bbbb --wait --output json ``` +Batch `--report` flags apply only to batch `--wait` reruns (`--all` or multiple test ids). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. When `--project` is omitted, the CLI infers `projectId` from polled run rows for classname / default suite naming; if inference fails, pass `--project ` explicitly (required under `--dry-run`). + Flags: - `--all` — rerun every test in the resolved project; requires `--project `. @@ -377,6 +397,7 @@ Flags: - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. - `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). +- `--report junit --report-file ` — with batch `--wait`, write a JUnit XML sidecar after polling (atomic write). Optional `--report-suite-name ` overrides the default `testsprite:` suite name. Requires `--wait`; not available on single-test reruns. A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 0a10df5..f33d08a 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -5,7 +5,7 @@ * sleep injection is wired through `TestDeps.sleep` to avoid real delays. */ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; @@ -3624,3 +3624,221 @@ describe('dashboardUrl on run completion', () => { ).toBe(true); }); }); + +describe('runTestRunAll — JUnit report export', () => { + const BATCH_FRESH_RESP: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + + function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { + return { + runId, + testId, + projectId: 'project_be', + userId: 'user_1', + status: status as RunResponse['status'], + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: '2026-06-09T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: status === 'passed' ? 3 : 0, + failedCount: 0, + }, + }; + } + + it('--wait --report junit writes XML after polling', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'junit-run-all-')); + const reportPath = join(dir, 'results.xml'); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_fresh_01') + return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; + if (runId === 'run_fresh_02') + return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'passed') }; + return errorBody('NOT_FOUND'); + }); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain(' { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'junit-run-fail-')); + const reportPath = join(dir, 'results.xml'); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_fresh_01') + return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; + if (runId === 'run_fresh_02') + return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'failed') }; + return errorBody('NOT_FOUND'); + }); + + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 1 }); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('failures="1"'); + expect(xml).toContain('name="test_be_02"'); + }); + + it('rejects --report without --wait', async () => { + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: './results.xml', + }, + {}, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('rejects --report-suite-name without --report', async () => { + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + reportSuiteName: 'orphan-suite', + }, + {}, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('--dry-run --report junit writes canned sample XML', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-')); + const reportPath = join(dir, 'results.xml'); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('name="test_fresh_wave_01"'); + expect(xml).toContain('failures="1"'); + }); + + it('--dry-run --report junit --report-suite-name overrides canned suite name', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-suite-')); + const reportPath = join(dir, 'results.xml'); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + reportSuiteName: 'ci-checkout-suite', + }, + { + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('. */ + reportSuiteName?: string; } /** @@ -5085,6 +5100,32 @@ interface RunTestRunAllOptions extends CommonOptions { maxConcurrency: number; /** Caller-supplied idempotency token; auto-minted if absent. */ idempotencyKey?: string; + /** --report junit: write a JUnit XML sidecar after batch --wait completes. */ + report?: JUnitReportFormat; + /** --report-file: destination path for the JUnit XML artifact. */ + reportFile?: string; + /** --report-suite-name: optional override for the JUnit . */ + reportSuiteName?: string; +} + +async function writeBatchJUnitReportIfRequested( + opts: { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + projectId?: string; + }, + results: readonly JUnitTestResult[], +): Promise { + if (opts.report !== 'junit' || opts.reportFile === undefined) return; + const projectId = resolveBatchReportProjectId(opts, results); + const suiteName = opts.reportSuiteName ?? `testsprite:${projectId}`; + const xml = buildJUnitReport({ + suiteName, + classname: projectId, + results, + }); + await writeJUnitReportFile(opts.reportFile, xml); } /** @@ -5093,6 +5134,8 @@ interface RunTestRunAllOptions extends CommonOptions { interface CliBatchRunFreshResult { testId: string; runId: string | undefined; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; status: string; error?: { code: string; message: string; exitCode: number }; /** CLIENT-synthesized Portal deep link (projectId from opts, testId per item). */ @@ -5119,6 +5162,13 @@ export async function runTestRunAll( ) { throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); } + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: true, + }); const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const out = makeOutput(opts.output, deps); @@ -5142,6 +5192,12 @@ export async function runTestRunAll( idempotencyKey, ...(opts.wait ? { thenPoll: '/api/cli/v1/runs/?waitSeconds=25' } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(opts.projectId, opts.reportSuiteName), + ); + } out.print(batchRunSample ?? envelope); return undefined; } @@ -5481,7 +5537,12 @@ export async function runTestRunAll( }, resolveAlternate, }); - return { testId: entry.testId, runId, status: finalRun.status }; + return { + testId: entry.testId, + runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -5563,6 +5624,7 @@ export async function runTestRunAll( total: pollable.length, }, }; + await writeBatchJUnitReportIfRequested(opts, freshRunResults); out.print(jsonPayload); // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7), @@ -5625,6 +5687,8 @@ export async function runTestRunAll( interface CliRerunResult { testId: string; runId: string; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; /** Terminal status, or 'timeout' for per-run deadline exceeded. */ status: string; /** Set when the test is a closure member (not the user's named test). */ @@ -5694,6 +5758,13 @@ export async function runTestRerun( } const isSingle = !opts.all && opts.testIds.length === 1; + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: !isSingle, + }); // ------------------------------------------------------------------------- // Pre-flight: auto-heal + Free-tier hint (best-effort, non-blocking) @@ -5733,6 +5804,13 @@ export async function runTestRerun( idempotencyKey, ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + const projectKey = resolveBatchReportProjectId(opts, []); + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(projectKey, opts.reportSuiteName), + ); + } out.print(findSample('POST', '/api/cli/v1/tests/batch/rerun')?.body() ?? envelope); } void client; @@ -6630,7 +6708,12 @@ export async function runTestRerun( }, resolveAlternate, }); - return { testId: entry.testId, runId: entry.runId, status: finalRun.status }; + return { + testId: entry.testId, + runId: entry.runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -6718,6 +6801,7 @@ export async function runTestRerun( total: accepted.length, }, }; + await writeBatchJUnitReportIfRequested(opts, rerunResults); out.print(jsonPayload); // Determine exit code: timeout (deferred or any timeout) → 7; any fail → 1; all pass → 0 @@ -7442,11 +7526,21 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--max-concurrency ', `with --all --wait, max in-flight polls at once (1-100, default: ${DEFAULT_BATCH_RUN_CONCURRENCY})`, ) + .option( + '--report ', + 'with --all --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + ' testsprite test run --all --project run all BE tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + '(see `testsprite test create --help` for details).', ) @@ -7476,6 +7570,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all --project .', ); } + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isAll, + }); if (isAll) { // --all path: wave-ordered fresh batch run. @@ -7506,6 +7608,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); @@ -7613,6 +7718,15 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--idempotency-key ', 'opaque key for safe retries (1–256 chars). Printed to stderr at --verbose if auto-generated.', ) + .option( + '--report ', + 'with batch --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nNotes:\n' + @@ -7638,10 +7752,20 @@ export function createTestCommand(deps: TestDeps = {}): Command { // `--no-auto-heal`. There is no explicit `--auto-heal` flag, so // autoHealExplicit is always false in this design — the default-on value // is never a deliberate user choice to opt in. + const testIds = testIdsArg ?? []; + const isBatch = cmdOpts.all === true || testIds.length !== 1; + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isBatch, + }); await runTestRerun( { ...resolveCommonOptions(command), - testIds: testIdsArg ?? [], + testIds, all: cmdOpts.all === true, projectId: cmdOpts.project, skipTerminal: cmdOpts.skipTerminal === true, @@ -7656,6 +7780,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); @@ -7679,6 +7806,9 @@ interface RunFlagOpts { project?: string; filter?: string; maxConcurrency?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface WaitFlagOpts { @@ -7697,6 +7827,9 @@ interface RerunFlagOpts { skipDependencies?: boolean; maxConcurrency?: string; idempotencyKey?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface UpdateFlagOpts { diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index e1bf8ff..a1838d6 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -1,5 +1,23 @@ import { describe, expect, it } from 'vitest'; -import { DRY_RUN_SAMPLE_ENTRIES, findSample } from './samples.js'; +import { DRY_RUN_SAMPLE_ENTRIES, findSample, sampleJUnitReportXml } from './samples.js'; + +describe('sampleJUnitReportXml', () => { + it('returns well-formed JUnit XML with canned batch ids', () => { + const xml = sampleJUnitReportXml('proj_dry'); + expect(xml).toContain(''); + expect(xml).toContain(' { + const xml = sampleJUnitReportXml('proj_dry', 'custom-ci-suite'); + expect(xml).toContain(' { it('resolves /me', () => { diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 28fa1bb..4c7cbb0 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -27,6 +27,7 @@ import type { CliTestStep, } from '../../commands/test.js'; import type { MeResponse } from '../../commands/auth.js'; +import { buildJUnitReport } from '../junit-report.js'; import type { Page } from '../pagination.js'; import type { TriggerRunResponse, @@ -67,6 +68,37 @@ const SAMPLE_REQUEST_ID = 'req_dry-run'; export const SAMPLE_DRY_RUN_REQUEST_ID = SAMPLE_REQUEST_ID; +/** + * Canned JUnit XML for batch `--wait --report junit --dry-run`. Mirrors the + * fresh batch-run sample ids so agents can learn the sidecar shape offline. + */ +export function sampleJUnitReportXml( + projectId: string = SAMPLE_PROJECT_ID, + reportSuiteName?: string, +): string { + return buildJUnitReport({ + suiteName: reportSuiteName ?? `testsprite:${projectId}`, + classname: projectId, + results: [ + { + testId: SAMPLE_TEST_ID_FRESH_1, + runId: SAMPLE_BATCH_FRESH_RUN_ID_1, + status: 'passed', + }, + { + testId: SAMPLE_TEST_ID_FRESH_2, + runId: SAMPLE_BATCH_FRESH_RUN_ID_2, + status: 'failed', + error: { + code: 'ASSERTION', + message: 'Expected checkout heading to be visible', + exitCode: 1, + }, + }, + ], + }); +} + const me: MeResponse = { userId: SAMPLE_USER_ID, keyId: SAMPLE_KEY_ID, diff --git a/src/lib/junit-report.test.ts b/src/lib/junit-report.test.ts new file mode 100644 index 0000000..d504dce --- /dev/null +++ b/src/lib/junit-report.test.ts @@ -0,0 +1,330 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError } from './errors.js'; +import { + assertJUnitReportOptions, + buildJUnitReport, + escapeXml, + parseJUnitReportFormat, + resolveBatchReportProjectId, + writeJUnitReportFile, + type JUnitTestResult, +} from './junit-report.js'; + +function makeResult(overrides: Partial & { testId: string }): JUnitTestResult { + return { + status: 'passed', + ...overrides, + }; +} + +describe('escapeXml', () => { + it('escapes XML special characters', () => { + expect(escapeXml(`a&bd"e'f`)).toBe('a&b<c>d"e'f'); + }); + + it('leaves plain text unchanged', () => { + expect(escapeXml('test_abc123')).toBe('test_abc123'); + }); +}); + +describe('parseJUnitReportFormat', () => { + it('accepts junit', () => { + expect(parseJUnitReportFormat('junit')).toBe('junit'); + }); + + it('returns undefined for absent value', () => { + expect(parseJUnitReportFormat(undefined)).toBeUndefined(); + expect(parseJUnitReportFormat('')).toBeUndefined(); + }); + + it('rejects unknown formats', () => { + expect(() => parseJUnitReportFormat('html')).toThrowError(ApiError); + try { + parseJUnitReportFormat('html'); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('assertJUnitReportOptions', () => { + it('allows absent report flags', () => { + expect(() => assertJUnitReportOptions({ wait: false, batchPath: true })).not.toThrow(); + }); + + it('rejects report-file without report', () => { + expect(() => + assertJUnitReportOptions({ reportFile: './out.xml', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); + + it('rejects report-suite-name without report', () => { + expect(() => + assertJUnitReportOptions({ + reportSuiteName: 'my-suite', + wait: true, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report on non-batch paths', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: true, + batchPath: false, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without wait', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: false, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without report-file', () => { + expect(() => + assertJUnitReportOptions({ report: 'junit', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); +}); + +describe('resolveBatchReportProjectId', () => { + it('prefers explicit projectId', () => { + expect(resolveBatchReportProjectId({ projectId: 'proj_a' }, [])).toBe('proj_a'); + }); + + it('infers from polled run rows', () => { + expect(resolveBatchReportProjectId({}, [{ projectId: 'proj_from_run' }])).toBe('proj_from_run'); + }); + + it('requires --project when the project cannot be inferred', () => { + expect(() => resolveBatchReportProjectId({}, [])).toThrowError(ApiError); + try { + resolveBatchReportProjectId({}, []); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('buildJUnitReport', () => { + it('renders an empty suite', () => { + const xml = buildJUnitReport({ + suiteName: 'Dry suite', + classname: 'proj_empty', + results: [], + }); + expect(xml).toContain( + ''); + }); + + it('counts passed tests without child elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [makeResult({ testId: 'test_a', status: 'passed' })], + }); + expect(xml).toContain(''); + expect(xml).not.toContain(' { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 'test_fail', + status: 'failed', + runId: 'run_1', + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('runId: run_1'); + expect(xml).toContain('failures="1"'); + }); + + it('maps blocked and cancelled to failures', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ testId: 't_blocked', status: 'blocked' }), + makeResult({ testId: 't_cancelled', status: 'cancelled' }), + ], + }); + expect(xml).toContain('failures="2"'); + expect(xml).toContain('type="blocked"'); + expect(xml).toContain('type="cancelled"'); + }); + + it('maps timeout to failure', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_timeout', + status: 'timeout', + error: { code: 'UNSUPPORTED', message: 'Timed out', exitCode: 7 }, + }), + ], + }); + expect(xml).toContain(''); + }); + + it('maps API error status to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_err', + status: 'error', + error: { code: 'NOT_FOUND', message: 'Run missing', exitCode: 4 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('errors="1"'); + }); + + it('maps auth failures to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_auth', + status: 'failed', + error: { code: 'AUTH_INVALID', message: 'Bad key', exitCode: 3 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('failures="0" errors="1"'); + }); + + it('escapes special characters in testcase names and messages', () => { + const xml = buildJUnitReport({ + suiteName: 'Suite "A"', + classname: 'proj<&>', + results: [ + makeResult({ + testId: 'test<1>', + status: 'failed', + error: { code: 'ASSERT', message: 'expected & "ok"', exitCode: 1 }, + }), + ], + }); + expect(xml).toContain('name="test<1>"'); + expect(xml).toContain('classname="proj<&>"'); + expect(xml).toContain('message="expected <true> & "ok""'); + }); + + it('aggregates mixed outcomes', () => { + const xml = buildJUnitReport({ + suiteName: 'Mixed', + classname: 'proj_mix', + results: [ + makeResult({ testId: 'p', status: 'passed' }), + makeResult({ testId: 'f', status: 'failed' }), + makeResult({ + testId: 'e', + status: 'error', + error: { code: 'INTERNAL', message: 'boom', exitCode: 10 }, + }), + ], + }); + expect(xml).toContain('tests="3" failures="1" errors="1" skipped="0"'); + }); +}); + +describe('writeJUnitReportFile', () => { + it('writes XML atomically to the target path', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-')); + const target = join(dir, 'results.xml'); + const xml = buildJUnitReport({ + suiteName: 'Suite', + classname: 'proj_write', + results: [makeResult({ testId: 't1', status: 'passed' })], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects a directory target', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-dir-')); + await expect(writeJUnitReportFile(dir, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects missing parent directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-parent-')); + const missing = join(dir, 'missing', 'out.xml'); + await expect(writeJUnitReportFile(missing, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('overwrites an existing file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-overwrite-')); + const target = join(dir, 'results.xml'); + writeFileSync(target, 'old', 'utf8'); + const xml = buildJUnitReport({ + suiteName: 'New', + classname: 'proj_new', + results: [], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects empty path', async () => { + await expect(writeJUnitReportFile('', '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + }); + + it('rejects parent that is a file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-file-parent-')); + const parentFile = join(dir, 'not-a-dir'); + writeFileSync(parentFile, 'x', 'utf8'); + const target = join(parentFile, 'out.xml'); + await expect(writeJUnitReportFile(target, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/src/lib/junit-report.ts b/src/lib/junit-report.ts new file mode 100644 index 0000000..3b11442 --- /dev/null +++ b/src/lib/junit-report.ts @@ -0,0 +1,267 @@ +import { createWriteStream } from 'node:fs'; +import { rename, stat, unlink } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { localValidationError, TransportError } from './errors.js'; + +export type JUnitReportFormat = 'junit'; + +/** Minimal testcase input shared by batch run and batch rerun poll results. */ +export interface JUnitTestResult { + testId: string; + runId?: string; + status: string; + /** Observed on polled runs; used for classname when --project is omitted. */ + projectId?: string; + error?: { code: string; message: string; exitCode?: number }; +} + +export interface JUnitReportBuildOptions { + suiteName: string; + classname: string; + results: readonly JUnitTestResult[]; +} + +export interface JUnitReportFlagOptions { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + wait: boolean; + /** True when the invocation is a batch path (run --all or rerun batch). */ + batchPath: boolean; +} + +const XML_DECL = ''; + +/** + * Parse `--report `. Only `junit` is accepted in v1. + */ +export function parseJUnitReportFormat(raw: string | undefined): JUnitReportFormat | undefined { + if (raw === undefined || raw === '') return undefined; + if (raw === 'junit') return 'junit'; + throw localValidationError('report', `unsupported report format "${raw}" — accepted: junit`); +} + +/** + * Validate `--report` / `--report-file` / `--report-suite-name` combinations. + * Report export is a sidecar artifact for batch `--wait` runs only. + */ +export function assertJUnitReportOptions(opts: JUnitReportFlagOptions): void { + if (opts.report === undefined) { + if (opts.reportFile !== undefined && opts.reportFile !== '') { + throw localValidationError('report-file', '--report-file requires --report junit'); + } + if (opts.reportSuiteName !== undefined && opts.reportSuiteName !== '') { + throw localValidationError( + 'report-suite-name', + '--report-suite-name requires --report junit', + ); + } + return; + } + + if (!opts.batchPath) { + throw localValidationError( + 'report', + '--report junit only applies to batch --wait runs (test run --all, or test rerun --all / multiple test ids)', + ); + } + if (!opts.wait) { + throw localValidationError( + 'report', + '--report junit requires --wait (the report is written after batch polling completes)', + ); + } + if (opts.reportFile === undefined || opts.reportFile === '') { + throw localValidationError('report-file', '--report junit requires --report-file '); + } +} + +/** + * Resolve the project id used for JUnit classname / default suite naming. + * Prefer explicit `--project`, then ids observed on polled run rows. + */ +export function resolveBatchReportProjectId( + opts: { projectId?: string }, + results: ReadonlyArray<{ projectId?: string }>, +): string { + if (opts.projectId) return opts.projectId; + const fromPoll = results.map(r => r.projectId).find((id): id is string => !!id); + if (fromPoll) return fromPoll; + throw localValidationError( + 'project', + '--report junit requires --project when the project cannot be inferred from run results', + ); +} + +/** + * Escape text for inclusion in XML element bodies and double-quoted attributes. + */ +export function escapeXml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +type JUnitOutcome = 'passed' | 'failure' | 'error' | 'skipped'; + +function classifyJUnitOutcome(status: string, error?: JUnitTestResult['error']): JUnitOutcome { + if (status === 'passed') return 'passed'; + if (status === 'skipped') return 'skipped'; + if (status === 'error' || error?.exitCode === 3) return 'error'; + return 'failure'; +} + +function failureMessage(result: JUnitTestResult): string { + if (result.error?.message) return result.error.message; + if (result.error?.code) return result.error.code; + return result.status; +} + +function renderTestcase(result: JUnitTestResult, classname: string): string { + const outcome = classifyJUnitOutcome(result.status, result.error); + const name = escapeXml(result.testId); + const cls = escapeXml(classname); + const lines = [` `]; + + if (outcome === 'failure') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.status); + const body = escapeXml( + [ + `status: ${result.status}`, + result.runId ? `runId: ${result.runId}` : undefined, + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'error') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.error?.code ?? result.status); + const body = escapeXml( + [ + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + result.runId ? `runId: ${result.runId}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'skipped') { + lines.push(` `); + } + + lines.push(' '); + return lines.join('\n'); +} + +/** + * Build a JUnit XML document from batch poll results. Duration is `0` in v1 + * because batch poll envelopes do not carry per-run timing. + */ +export function buildJUnitReport(opts: JUnitReportBuildOptions): string { + const results = opts.results; + let failures = 0; + let errors = 0; + let skipped = 0; + + for (const result of results) { + const outcome = classifyJUnitOutcome(result.status, result.error); + if (outcome === 'failure') failures++; + else if (outcome === 'error') errors++; + else if (outcome === 'skipped') skipped++; + } + + const suiteName = escapeXml(opts.suiteName); + const testcases = results.map(r => renderTestcase(r, opts.classname)).join('\n'); + + return [ + XML_DECL, + '', + ` `, + testcases, + ' ', + '', + '', + ].join('\n'); +} + +async function assertReportFileParent(rawPath: string): Promise { + if (typeof rawPath !== 'string' || rawPath.length === 0) { + throw localValidationError('report-file', 'must be a non-empty file path'); + } + const resolved = isAbsolute(rawPath) ? rawPath : resolve(process.cwd(), rawPath); + if (resolved.endsWith('/') || resolved.endsWith('\\')) { + throw localValidationError('report-file', 'must point to a file, not a directory'); + } + + const parent = dirname(resolved); + let parentStat; + try { + parentStat = await stat(parent); + } catch { + throw localValidationError('report-file', `parent directory does not exist: ${parent}`); + } + if (!parentStat.isDirectory()) { + throw localValidationError('report-file', `parent path is not a directory: ${parent}`); + } + + let targetStat; + try { + targetStat = await stat(resolved); + } catch { + return resolved; + } + if (targetStat.isDirectory()) { + throw localValidationError('report-file', `must point to a file, not a directory: ${resolved}`); + } + return resolved; +} + +/** + * Atomically write JUnit XML to `--report-file` (temp sibling + rename). + */ +export async function writeJUnitReportFile(rawPath: string, xml: string): Promise { + const resolved = await assertReportFileParent(rawPath); + const parent = dirname(resolved); + const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); + + await new Promise((resolvePromise, reject) => { + const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); + let streamError: Error | null = null; + stream.on('error', err => { + streamError = err instanceof Error ? err : new Error(String(err)); + }); + stream.write(xml, err => { + if (err) { + streamError = err instanceof Error ? err : new Error(String(err)); + } + stream.end(() => { + if (streamError) { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError(`Failed to write --report-file ${resolved}: ${streamError.message}`), + ); + return; + } + rename(tmpPath, resolved) + .then(() => resolvePromise()) + .catch(renameErr => { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError( + `Failed to write --report-file ${resolved}: ${renameErr instanceof Error ? renameErr.message : String(renameErr)}`, + ), + ); + }); + }); + }); + }); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6fee8af..9e5e0d9 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -392,32 +392,40 @@ Exit codes: On failure/blocked/cancelled, run: testsprite test artifact get Options: - --all rerun all tests in the resolved project (requires - --project) (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --skip-terminal with --all: skip tests already in a terminal status - (passed|failed|blocked|cancelled) (default: false) - --status with --all: only dispatch tests whose status matches - one of these values (comma-separated; accepted: - draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) - --filter with --all: only rerun tests whose name contains - this substring (case-insensitive) - --wait block until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --no-auto-heal opt out of AI heal-on-drift for this FE rerun - (default: auto-heal is ON). Costs 0.2 credits per - engage when a step has drifted. Ignored for backend - tests. - --skip-dependencies BE only: rerun only the named test without expanding - the producer/teardown closure (default: false) - --max-concurrency with --wait, max in-flight polls at once (1-100, - default: 50) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --verbose if auto-generated. - -h, --help display help for command + --all rerun all tests in the resolved project (requires + --project) (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --skip-terminal with --all: skip tests already in a terminal + status (passed|failed|blocked|cancelled) + (default: false) + --status with --all: only dispatch tests whose status + matches one of these values (comma-separated; + accepted: + draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) + --filter with --all: only rerun tests whose name contains + this substring (case-insensitive) + --wait block until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --no-auto-heal opt out of AI heal-on-drift for this FE rerun + (default: auto-heal is ON). Costs 0.2 credits per + engage when a step has drifted. Ignored for + backend tests. + --skip-dependencies BE only: rerun only the named test without + expanding the producer/teardown closure (default: + false) + --max-concurrency with --wait, max in-flight polls at once (1-100, + default: 50) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --verbose if auto-generated. + --report with batch --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Notes: • rerun replays a saved run/script and is MORE LENIENT than a fresh \`test run\` @@ -490,28 +498,34 @@ Exit codes: On failure/blocked/cancelled, run: testsprite test artifact get Options: - --target-url override the project default env URL for this run - (http/https only, no localhost/private IPs) - --wait poll until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --debug if auto-generated. - --all run all BE tests in the project (wave-ordered fresh - run; requires --project). Mutually exclusive with - . (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --filter with --all: only run tests whose name contains this - substring (case-insensitive) - --max-concurrency with --all --wait, max in-flight polls at once - (1-100, default: 50) - -h, --help display help for command + --target-url override the project default env URL for this run + (http/https only, no localhost/private IPs) + --wait poll until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --debug if auto-generated. + --all run all BE tests in the project (wave-ordered + fresh run; requires --project). Mutually + exclusive with . (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --filter with --all: only run tests whose name contains + this substring (case-insensitive) + --max-concurrency with --all --wait, max in-flight polls at once + (1-100, default: 50) + --report with --all --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Dependency-aware fresh run (M4): testsprite test run --all --project run all BE tests in wave order testsprite test run --all --project --filter name-glob subset + testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). From 38e0f190666267e2c9b5d526424f7b5a9682db0e Mon Sep 17 00:00:00 2001 From: Rahul Joshi <186129212+crypticsaiyan@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:02:06 +0530 Subject: [PATCH 035/117] fix(rerun): preserve exit code and escalate auth errors in batch rerun (#33) pollAccepted in runTestRerun hardcoded exitCode:1 on ApiError, causing the auth-escalation find(r => r.error?.exitCode === 3) to always return undefined -- auth failures silently exited 1 instead of 3. - preserve err.exitCode in pollAccepted (mirrors runTestRunAll fix) - add auth escalation block before generic exit-1 throw - bound initial chunk idempotency key to <=256 chars (mirrors retry path) --- src/commands/test.rerun.spec.ts | 482 ++++++++++++++++++++++++++++++++ src/commands/test.ts | 26 +- 2 files changed, 506 insertions(+), 2 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index acd5b02..d36a233 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4698,3 +4698,485 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); + +// --------------------------------------------------------------------------- +// [fix-exitcode] pollAccepted preserves ApiError exit codes (not hardcoded 1) +// --------------------------------------------------------------------------- + +describe('[fix-exitcode] polling error exit codes preserved in batch rerun results', () => { + it('AUTH_REQUIRED during polling → batch escalates to exitCode 3', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a1', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_auth_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_auth_fail')) return errorBody('AUTH_REQUIRED'); + if (url.includes('/runs/run_pass_a1')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e as { exitCode?: number; message?: string }); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + }); + + it('RATE_LIMITED during polling → non-auth, batch exits 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a2', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_rl', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_rl')) return errorBody('RATE_LIMITED'); + if (url.includes('/runs/run_pass_a2')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); + + it('NOT_FOUND during run polling → non-auth, batch exits 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a3', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_nf', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_nf')) return errorBody('NOT_FOUND'); + if (url.includes('/runs/run_pass_a3')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// [fix-auth-escalation] batch auth failure escalates to exit 3 +// --------------------------------------------------------------------------- + +describe('[fix-auth-escalation] auth error in batch rerun polling escalates to exit 3', () => { + it('auth failure in batch poll → batch exits 3, not 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_other', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_auth', runId: 'run_auth', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_other', runId: 'run_other', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_auth')) return errorBody('AUTH_REQUIRED'); + if (url.includes('/runs/run_other')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_auth', 'test_other'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + }); + + it('mixed batch: one pass, one auth failure → exits 3 (auth wins)', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_pass', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_auth2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const passRun = makeTerminalRun('run_pass', 'passed'); + passRun.testId = 'test_1'; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_pass')) return { body: passRun }; + if (url.includes('/runs/run_auth2')) return errorBody('AUTH_REQUIRED'); + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + expect((err as { message?: string }).message).toMatch(/auth error/i); + }); + + it('non-auth failure → exits 1 (no escalation)', async () => { + const creds = makeCreds(); + const failRun = makeTerminalRun('run_fail', 'failed'); + failRun.testId = 'test_1'; + const passRun = makeTerminalRun('run_pass_c3', 'passed'); + passRun.testId = 'test_2'; + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_c3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_fail')) return { body: failRun }; + if (url.includes('/runs/run_pass_c3')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// [fix-D4] initial chunk idempotency key bounded to ≤256 chars +// --------------------------------------------------------------------------- + +describe('[fix-D4] initial chunk dispatch idempotency key bounded to 256 chars', () => { + it('short key with multiple chunks passes through unchanged', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // 51 test IDs forces 2 chunks (MAX_BATCH_RERUN_IDS = 50) + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + const key = h.get('idempotency-key') ?? ''; + receivedKeys.push(key); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: 'short-key', + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + expect(receivedKeys[0]).toBe('short-key:chunk0'); + expect(receivedKeys[1]).toBe('short-key:chunk1'); + expect(receivedKeys[0]!.length).toBeLessThanOrEqual(256); + expect(receivedKeys[1]!.length).toBeLessThanOrEqual(256); + }); + + it('249-char key + :chunk0 suffix would exceed 256 → key truncated to keep total ≤256', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // key is 249 chars; `:chunk0` is 7 chars → 256 total (edge case, fits exactly) + const longKey = 'k'.repeat(249); + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + receivedKeys.push(h.get('idempotency-key') ?? ''); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: longKey, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + for (const key of receivedKeys) { + expect(key.length).toBeLessThanOrEqual(256); + } + // suffix must be preserved + expect(receivedKeys[0]).toMatch(/:chunk0$/); + expect(receivedKeys[1]).toMatch(/:chunk1$/); + }); + + it('256-char key + :chunk0 suffix → base truncated so total is exactly 256', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // Max-length user key: 256 chars. `:chunk0` = 7 chars → need to truncate base to 249. + const maxKey = 'x'.repeat(256); + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + receivedKeys.push(h.get('idempotency-key') ?? ''); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: maxKey, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + for (const key of receivedKeys) { + expect(key.length).toBeLessThanOrEqual(256); + } + expect(receivedKeys[0]).toMatch(/:chunk0$/); + expect(receivedKeys[1]).toMatch(/:chunk1$/); + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index 6c7fe65..a48812e 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -6377,7 +6377,15 @@ export async function runTestRerun( chunkResponses = []; for (let idx = 0; idx < chunks.length; idx++) { const chunk = chunks[idx]!; - const chunkKey = chunks.length === 1 ? idempotencyKey : `${idempotencyKey}:chunk${idx}`; + // Bound the per-chunk idempotency key to <=256 chars (mirrors the retry + // path). A long base key plus the `:chunkN` suffix could otherwise exceed + // the server cap and be rejected or truncated inconsistently. + const chunkSuffix = chunks.length === 1 ? '' : `:chunk${idx}`; + const chunkBase = + chunkSuffix.length > 0 && idempotencyKey.length + chunkSuffix.length > 256 + ? idempotencyKey.slice(0, 256 - chunkSuffix.length) + : idempotencyKey; + const chunkKey = `${chunkBase}${chunkSuffix}`; const chunkResp = await client.triggerBatchRerun( { source: 'cli', @@ -6728,11 +6736,14 @@ export async function runTestRerun( }; } if (err instanceof ApiError) { + // Preserve the real exit code (AUTH_INVALID=3, RATE_LIMITED=11, …) so the + // batch exit-code aggregator can escalate auth failures correctly. Mirroring + // the identical fix already applied to runTestRunAll's pollFreshAccepted. return { testId: entry.testId, runId: entry.runId, status: 'error', - error: { code: err.code, message: err.message, exitCode: 1 }, + error: { code: err.code, message: err.message, exitCode: err.exitCode }, }; } throw err; @@ -6834,6 +6845,17 @@ export async function runTestRerun( } if (failed > 0) { + // Auth failure on any member is a batch-wide condition — the credential is + // bad, not the test. Propagate exit 3 so the operator fixes auth rather than + // chasing a "rerun failed" (exit 1). Mirrors the identical logic already + // applied to runTestRunAll lines 5462-5468. + const authErr = rerunResults.find(r => r.error?.exitCode === 3); + if (authErr) { + throw new CLIError( + `${failed} rerun${failed !== 1 ? 's' : ''} failed — auth error (${authErr.error?.code}): ${authErr.error?.message}`, + 3, + ); + } throw new CLIError(`${failed} rerun${failed !== 1 ? 's' : ''} failed.`, 1); } From e1d00f1816cd87fd2f64337708f2ed8849456efa Mon Sep 17 00:00:00 2001 From: Resque Date: Sun, 5 Jul 2026 23:32:23 +0400 Subject: [PATCH 036/117] feat(agent): add kiro as an install target (#10) * feat(agent): add kiro as an install target Adds kiro as an own-file agent target on the current multi-skill model: AgentTarget union, a pathFor('kiro') case landing at .kiro/skills//SKILL.md, and a TARGETS entry (experimental, own-file, wrapSkill frontmatter like claude/antigravity). Kiro installs both default skills (testsprite-verify + testsprite-onboard). Updates the --target help string, README/DOCUMENTATION target lists and counts, unit + command tests (six keys, list 12 rows, five own-file targets, 10 dry-run would-write lines, renderForTarget + content-integrity coverage), and regenerates the agent/setup help snapshots. Rebuilt on current main. * test(e2e): include kiro in target matrix guards and multi-target install The Local E2E Tests CI job failed because two matrix-coverage guards hardcoded the target set (claude, antigravity, cursor, cline, codex) and did not include the new kiro target. Add kiro (own-file, between cline and codex to match TARGETS order) to both guards, and add kiro to the multi-target install e2e so the target is actually exercised end-to-end. --- DOCUMENTATION.md | 7 ++--- README.md | 2 +- src/commands/agent.test.ts | 25 +++++++++-------- src/commands/agent.ts | 2 +- src/lib/agent-targets.test.ts | 27 ++++++++++++++++--- src/lib/agent-targets.ts | 12 ++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 7 ++--- test/e2e/agent-install.e2e.test.ts | 15 ++++++++--- test/e2e/setup.e2e.test.ts | 9 ++++++- 9 files changed, 77 insertions(+), 29 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index a1ac967..7a89216 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -113,14 +113,15 @@ testsprite agent install codex # install into AGENTS.md for Codex (managed- testsprite agent install cursor # .cursor/rules/testsprite-verify.mdc testsprite agent install cline # .clinerules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md -testsprite agent list # list all 5 targets with status + mode + path +testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md +testsprite agent list # list all 6 targets with status + mode + path ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental). +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental). The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro) backs up the existing file to `.bak` first. ## Command reference diff --git a/README.md b/README.md index 067a620..1e20a1d 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- | | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | | | `test wait` | Block on a `runId` until terminal | | | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 9bdbdf1..493cc4e 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -741,6 +741,7 @@ describe('runList', () => { expect(out).toContain('cursor'); expect(out).toContain('cline'); expect(out).toContain('antigravity'); + expect(out).toContain('kiro'); expect(out).toContain('codex'); expect(out).toContain('ga'); expect(out).toContain('experimental'); @@ -749,6 +750,7 @@ describe('runList', () => { expect(out).toContain(TARGETS.cursor.path); expect(out).toContain(TARGETS.cline.path); expect(out).toContain(TARGETS.antigravity.path); + expect(out).toContain(TARGETS.kiro.path); expect(out).toContain(TARGETS.codex.path); }); @@ -759,13 +761,14 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 5 targets × 2 default skills = 10 rows - expect(json).toHaveLength(10); + // 6 targets × 2 default skills = 12 rows + expect(json).toHaveLength(12); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); expect(targets).toContain('antigravity'); + expect(targets).toContain('kiro'); expect(targets).toContain('codex'); // skill field present on each row const skills = json.map(r => r.skill); @@ -905,11 +908,11 @@ describe('createAgentCommand wiring', () => { }); // --------------------------------------------------------------------------- -// All four own-file targets installed at once +// All five own-file targets installed at once // --------------------------------------------------------------------------- -describe('runInstall — all four own-file targets', () => { - it('installs all four own-file targets in one invocation', async () => { +describe('runInstall — all five own-file targets', () => { + it('installs all five own-file targets in one invocation', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -919,7 +922,7 @@ describe('runInstall — all four own-file targets', () => { output: 'text', debug: false, dryRun: false, - target: ['claude', 'cursor', 'cline', 'antigravity'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro'], skills: ['testsprite-verify'], force: false, }, @@ -937,11 +940,11 @@ describe('runInstall — all four own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all four own-file targets +// Dry-run for all five own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the four own-file targets (default 2 skills = 8 would-write lines)', async () => { + it('writes nothing for any of the five own-file targets (default 2 skills = 10 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -951,7 +954,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -961,9 +964,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 4 targets × 2 default skills = 8 would-write lines + // 5 targets × 2 default skills = 10 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(8); + expect(wouldWriteLines.length).toBe(10); }); }); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index ebde317..8213755 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -809,7 +809,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, codex (comma-separated or repeated)', collect, [], ) diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index b049711..8ef0e36 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -74,19 +74,20 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all five required keys', () => { + it('has all six required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor']); + expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro']); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, antigravity, and codex are experimental', () => { + it('cursor, cline, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); + expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); }); @@ -102,6 +103,7 @@ describe('TARGETS', () => { expect(TARGETS.antigravity.mode).toBe('own-file'); expect(TARGETS.cursor.mode).toBe('own-file'); expect(TARGETS.cline.mode).toBe('own-file'); + expect(TARGETS.kiro.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -200,6 +202,22 @@ describe('renderForTarget("antigravity")', () => { }); }); +describe('renderForTarget("kiro")', () => { + const result = renderForTarget('kiro', 'testsprite-verify', STUB_BODY); + + it('returns the correct path', () => { + expect(result.path).toBe('.kiro/skills/testsprite-verify/SKILL.md'); + }); + + it('frontmatter contains name: testsprite-verify', () => { + expect(result.content).toContain('name: testsprite-verify'); + }); + + it('frontmatter contains description:', () => { + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); +}); + describe('renderForTarget("claude") vs renderForTarget("antigravity")', () => { it('produce the same frontmatter lines (name + description)', () => { const claude = renderForTarget('claude', 'testsprite-verify', STUB_BODY); @@ -274,11 +292,12 @@ describe('renderForTarget("cline")', () => { // --------------------------------------------------------------------------- describe('content integrity — own-file targets', () => { - const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity'> = [ + const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity' | 'kiro'> = [ 'claude', 'cursor', 'cline', 'antigravity', + 'kiro', ]; // Use the real body for these checks, since we're guarding against trimming. diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 2547daf..663a998 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -1,6 +1,6 @@ import { readFileSync } from 'node:fs'; -export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex'; +export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex' | 'kiro'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -140,6 +140,8 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.cursor/rules/${skill}.mdc`; case 'cline': return `.clinerules/${skill}.md`; + case 'kiro': + return `.kiro/skills/${skill}/SKILL.md`; case 'codex': return 'AGENTS.md'; } @@ -170,6 +172,14 @@ export const TARGETS: Record = { mode: 'own-file', wrap: (_name, _description, body) => body, }, + kiro: { + status: 'experimental', + path: pathFor('kiro', SKILL_NAME), + mode: 'own-file', + // kiro reads SKILL.md files with name/description frontmatter, same as + // claude/antigravity, so it shares the wrapSkill wrapper. + wrap: wrapSkill, + }, /** * codex target — managed-section mode. * diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 9e5e0d9..03cb551 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -25,8 +25,8 @@ Write the TestSprite agent skills (verification loop + first-run onboarding) into a project for a coding agent Options: - --target Agent target(s): claude, cursor, cline, antigravity, codex - (comma-separated or repeated) (default: []) + --target Agent target(s): claude, cursor, cline, antigravity, kiro, + codex (comma-separated or repeated) (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -112,7 +112,8 @@ Options: --from-env Read TESTSPRITE_API_KEY from the environment instead of prompting (default: false) --agent Coding-agent target to install: claude, antigravity, - cursor, cline, codex (default: claude) (default: "claude") + cursor, cline, kiro, codex (default: claude) (default: + "claude") --no-agent Skip the agent skill install (configure credentials only) --force Overwrite an existing skill file (a .bak backup is kept) --dir Project root for the skill install (default: current diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 1cbf4fc..0513b8b 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -430,13 +430,13 @@ describe('dry-run', () => { // --------------------------------------------------------------------------- describe('multi-target install', () => { - it('--target=claude,cursor,cline,antigravity,codex writes all targets + skills, exit 0', () => { + it('--target=claude,cursor,cline,antigravity,kiro,codex writes all targets + skills, exit 0', () => { const tmpDir = freshTmpDir(); const result = runCli([ 'agent', 'install', - '--target=claude,cursor,cline,antigravity,codex', + '--target=claude,cursor,cline,antigravity,kiro,codex', '--dir', tmpDir, '--output', @@ -449,7 +449,7 @@ describe('multi-target install', () => { action: string; path: string; }>; - const allTargets: AgentTarget[] = ['claude', 'cursor', 'cline', 'antigravity', 'codex']; + const allTargets: AgentTarget[] = ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'codex']; for (const target of allTargets) { if (TARGETS[target].mode === 'managed-section') { @@ -819,7 +819,14 @@ describe('agent list', () => { // --------------------------------------------------------------------------- describe('matrix coverage guard', () => { it('TARGETS matches the documented, e2e-covered set (update this list when adding a target)', () => { - expect(Object.keys(TARGETS)).toEqual(['claude', 'antigravity', 'cursor', 'cline', 'codex']); + expect(Object.keys(TARGETS)).toEqual([ + 'claude', + 'antigravity', + 'cursor', + 'cline', + 'kiro', + 'codex', + ]); }); it('SKILLS matches the documented, e2e-covered set (update this list when adding a skill)', () => { diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index b5d3963..a30c3fb 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -222,7 +222,14 @@ describe('deprecated `init` alias', () => { describe('matrix coverage guard', () => { it('TARGETS matches the documented set (update this list when adding a target)', () => { - expect(Object.keys(TARGETS)).toEqual(['claude', 'antigravity', 'cursor', 'cline', 'codex']); + expect(Object.keys(TARGETS)).toEqual([ + 'claude', + 'antigravity', + 'cursor', + 'cline', + 'kiro', + 'codex', + ]); }); }); From 3fd703ffac59a38b6b5c227befdaf2510497b308 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:32:44 +0200 Subject: [PATCH 037/117] fix(poll): emit partial run to stdout on TimeoutError in run --wait and test wait (#153) Apply the fix in src/commands/ (the compiled CLI path). When the overall --timeout polling deadline is exceeded, emit {runId, status:"running"} to stdout before exit 7 so JSON agents can chain into test wait. Co-authored-by: Contributor Co-authored-by: Cursor --- src/commands/test.run.spec.ts | 68 ++++++++++++++++++++++++++++++++++ src/commands/test.ts | 32 ++++++++++++++++ src/commands/test.wait.spec.ts | 63 +++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+) diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index f33d08a..2f2ea9d 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -1935,6 +1935,74 @@ describe('runTestRun --wait: Fix 3 — RequestTimeoutError writes partial JSON t }); }); +// --------------------------------------------------------------------------- +// TimeoutError on --wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestRun --wait: TimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const { credentialsPath } = makeCreds(); + let dateCallCount = 0; + let fetchCallCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (++dateCallCount > 6 ? base + 2000 : base); + + try { + const fetchImpl: typeof globalThis.fetch = async () => { + ++fetchCallCount; + if (fetchCallCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + const runningRun: RunResponse = { ...makePassedRun(), status: 'running' }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + targetUrl: string; + }; + expect(stdoutJson.runId).toBe(TRIGGER_RESP.runId); + expect(stdoutJson.status).toBe('running'); + expect(stdoutJson.targetUrl).toBe(TRIGGER_RESP.targetUrl); + } finally { + Date.now = realDateNow; + } + }); +}); + // --------------------------------------------------------------------------- // Fix 5 — B2(c): --timeout hint fires on default, not on explicit timeout // --------------------------------------------------------------------------- diff --git a/src/commands/test.ts b/src/commands/test.ts index a48812e..29cffd5 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -4861,6 +4861,26 @@ export async function runTestRun( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${triggerResponse.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, timeoutPartial, opts.createContext, data => { + const p = data as typeof timeoutPartial; + const lines = [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + ]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + return lines.join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md @@ -5021,6 +5041,18 @@ export async function runTestWait( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${opts.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { runId: opts.runId, status: 'running' as const }; + out.print(timeoutPartial, data => { + const p = data as typeof timeoutPartial; + return [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md diff --git a/src/commands/test.wait.spec.ts b/src/commands/test.wait.spec.ts index 68a4c99..c83195d 100644 --- a/src/commands/test.wait.spec.ts +++ b/src/commands/test.wait.spec.ts @@ -1021,6 +1021,69 @@ describe('runTestWait: Fix 3 — RequestTimeoutError writes partial JSON to stdo }); }); +// --------------------------------------------------------------------------- +// TimeoutError on test wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestWait: TimeoutError writes partial JSON to stdout', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('running') })); + + let callCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (callCount++ > 4 ? base + 2000 : base); + + try { + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + } finally { + Date.now = realDateNow; + } + }); +}); + // --------------------------------------------------------------------------- // FIX 4 — D5-UX: text mode shows the `error` string for failed/blocked runs // --------------------------------------------------------------------------- From dcbcbee16c868e8fb96dc84824d0596e6f1c2703 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:33:11 -0400 Subject: [PATCH 038/117] fix(steps): surface run-scoped per-step error text and stepType (#167) --- src/commands/test.test.ts | 59 +++++++++++++++++++++++++++++++++++++++ src/commands/test.ts | 44 +++++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index d3b45ca..501ed50 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -2603,6 +2603,65 @@ describe('runSteps', () => { expect(step2.outcomeContributesToFailure).toBe(true); }); + it('--run-id carries the per-step error text and stepType through to JSON (no silent drop)', async () => { + // Regression lock for the "steps discard RunStepDto.error" gap: the wire + // already returns the failure text in the same response; it must survive + // the CliTestStep mapping instead of forcing an artifact-bundle download. + const { credentialsPath } = makeCreds(); + const runWithStepError = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + steps: [ + RUN_WITH_STEPS.steps[0]!, + { + ...RUN_WITH_STEPS.steps[1]!, + status: 'failed', + error: 'Expected heading "Order confirmed" to be visible, got hidden', + }, + ], + }; + const fetchImpl = makeFetch(() => ({ body: runWithStepError })); + const page = await runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + const passing = page.items.find(s => s.stepIndex === 1)!; + const failing = page.items.find(s => s.stepIndex === 2)!; + expect(failing.error).toBe('Expected heading "Order confirmed" to be visible, got hidden'); + expect(failing.stepType).toBe('assertion'); + expect(passing.error).toBeNull(); + expect(passing.stepType).toBe('action'); + }); + + it('--run-id text mode prints an indented error: sub-line under the failed row only', async () => { + const { credentialsPath } = makeCreds(); + const runWithStepError = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + steps: [ + RUN_WITH_STEPS.steps[0]!, + { + ...RUN_WITH_STEPS.steps[1]!, + status: 'failed', + error: 'Locator resolved to hidden element\n at assert heading', + }, + ], + }; + const fetchImpl = makeFetch(() => ({ body: runWithStepError })); + const out: string[] = []; + await runSteps( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + // Newlines in the wire error collapse to one displayable line. + expect(block).toContain('error: Locator resolved to hidden element at assert heading'); + // Exactly one sub-line: the passing step must not grow one. + expect(block.match(/error: /g)).toHaveLength(1); + }); + it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => { const { credentialsPath } = makeCreds(); // The run-scoped endpoint returns a run whose testId differs from the diff --git a/src/commands/test.ts b/src/commands/test.ts index 29cffd5..ede6376 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -186,6 +186,19 @@ export interface CliTestStep { * that don't emit the field still type-check. */ outcomeContributesToFailure?: boolean | null; + /** + * Per-step failure text, carried from `RunStepDto.error` on the run-scoped + * endpoint (`GET /runs/{id}?includeSteps=true`). Only present on `--run-id` + * responses; the cumulative `/tests/{id}/steps` rows do not carry it, so the + * field stays optional (additive, non-breaking for existing consumers). + */ + error?: string | null; + /** + * Wire step kind from `RunStepDto.type` on the run-scoped endpoint. Same + * availability rules as `error`. Named `stepType` to avoid colliding with + * the free-form `action` label above. + */ + stepType?: 'action' | 'assertion'; } /** @@ -3642,6 +3655,12 @@ function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestSte // non-contributors. (Per the CliTestStep contract: null ≠ false.) outcomeContributesToFailure: run.failedStepIndex === null ? null : numericIndex === run.failedStepIndex, + // Carry the per-step failure text and the wire step kind through instead + // of dropping them: the agent asking "why did this step fail?" would + // otherwise have to download the whole artifact bundle to read a string + // this very response already contained. + error: step.error, + stepType: step.type, }; } @@ -3970,6 +3989,14 @@ const RUN_HISTORY_TABLE_COL_WIDTHS = { */ const DESC_COL_MAX = 60; +/** + * Cap, in chars, for the one-line `error:` sub-line under a failed step row + * in `renderStepsText`. Long enough for a full assertion message, short + * enough that a stack-trace blob can't flood the table. Full text is in + * `--output json`. + */ +const ERROR_SUBLINE_MAX = 200; + /** Max chars to show in the TARGETURL sub-line (excess truncated with …). */ const HISTORY_TARGET_URL_MAX = 80; @@ -8433,9 +8460,9 @@ function renderStepsText(page: Page): string { ' ' + 'UPDATED'; - const rows = page.items.map(s => { + const rows = page.items.flatMap(s => { const marker = s.outcomeContributesToFailure === true ? '* ' : ' '; - return [ + const row = [ marker, pad(String(s.stepIndex), indexWidth), pad(s.action, actionWidth), @@ -8443,6 +8470,19 @@ function renderStepsText(page: Page): string { pad(descOf(s), descWidth), s.updatedAt, ].join(' '); + // Run-scoped rows carry the per-step failure text; surface it as an + // indented sub-line under failed rows (mirrors the history table's + // `targetUrl:` sub-line). Collapsed to one line and capped so a huge + // stack blob can't wreck the table; full text ships in --output json. + if (s.status === 'failed' && typeof s.error === 'string' && s.error.length > 0) { + const oneLine = s.error.replace(/\s+/g, ' ').trim(); + const shown = + oneLine.length > ERROR_SUBLINE_MAX + ? `${oneLine.slice(0, ERROR_SUBLINE_MAX - 1)}…` + : oneLine; + return [row, ` error: ${shown}`]; + } + return [row]; }); const lines: string[] = [header, ...rows, '']; From 9457583f27b8c4e57541933db9ed45e554dc9ad2 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:33:26 -0400 Subject: [PATCH 039/117] feat(test): add "test diff " to isolate run-to-run regressions (#168) * feat(test): add "test diff " to isolate run-to-run regressions * test(diff): cover runDiff --dry-run branch (offline canned sample) --- src/commands/test.test.ts | 127 +++++++++++ src/commands/test.ts | 200 ++++++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 5 + 3 files changed, 332 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 501ed50..38c16e5 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -27,6 +27,7 @@ import { runCreateBatch, runCreateFromPlan, runDelete, + runDiff, runFailureGet, runFailureSummary, runGet, @@ -121,6 +122,7 @@ describe('createTestCommand — surface', () => { 'create-batch', 'delete', 'delete-batch', + 'diff', 'failure', 'get', 'list', @@ -2833,6 +2835,131 @@ describe('runSteps', () => { }); }); +describe('runDiff', () => { + const baseRun = { + testId: 'test_fe', + projectId: 'project_alice', + userId: 'u1', + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + targetUrl: 'https://example.com', + createdFrom: null, + error: null, + videoUrl: null, + stepSummary: { total: 2, completed: 2, passedCount: 2, failedCount: 0 }, + }; + const makeStep = (index: string, status: string, error: string | null = null) => ({ + stepIndex: index, + type: 'action', + action: `step ${index}`, + status, + description: `Step ${index}`, + error, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-06-01T10:00:05.000Z', + }); + const RUN_GREEN = { + ...baseRun, + runId: 'run_green', + status: 'passed', + codeVersion: 'v1', + failedStepIndex: null, + failureKind: null, + steps: [makeStep('0001', 'passed'), makeStep('0002', 'passed')], + }; + const RUN_RED = { + ...baseRun, + runId: 'run_red', + status: 'failed', + codeVersion: 'v2', + failedStepIndex: 2, + failureKind: 'assertion', + steps: [makeStep('0001', 'passed'), makeStep('0002', 'failed', 'heading not visible')], + }; + const fetchForRuns = () => + makeFetch(url => ({ body: url.includes('run_green') ? RUN_GREEN : RUN_RED })); + + it('reports the verdict flip, the changed step with its error, and code drift, then exits 1', async () => { + const { credentialsPath } = makeCreds(); + const out: string[] = []; + const rejection = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' }, + { credentialsPath, fetchImpl: fetchForRuns(), stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const printed = JSON.parse(out.join('')) as { + verdictChanged: boolean; + codeVersionChanged: boolean; + crossTest: boolean; + changedSteps: Array<{ stepIndex: number; statusA: string; statusB: string; errorB?: string }>; + }; + expect(printed.verdictChanged).toBe(true); + expect(printed.codeVersionChanged).toBe(true); + expect(printed.crossTest).toBe(false); + expect(printed.changedSteps).toHaveLength(1); + expect(printed.changedSteps[0]).toMatchObject({ + stepIndex: 2, + statusA: 'passed', + statusB: 'failed', + errorB: 'heading not visible', + }); + }); + + it('identical verdicts resolve with exit 0 and no step changes', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RUN_GREEN })); + const diff = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_green' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(diff.verdictChanged).toBe(false); + expect(diff.changedSteps).toHaveLength(0); + }); + + it('warns (not fails) when the runs belong to different tests', async () => { + const { credentialsPath } = makeCreds(); + const OTHER = { ...RUN_GREEN, runId: 'run_red', testId: 'test_other' }; + const fetchImpl = makeFetch(url => ({ + body: url.includes('run_green') ? RUN_GREEN : OTHER, + })); + const errs: string[] = []; + const diff = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) }, + ); + expect(diff.crossTest).toBe(true); + expect(errs.join('\n')).toContain('different tests'); + }); + + it('--dry-run returns the canned sample fully offline (no credentials, no fetch)', async () => { + // Dry-run must not require credentials or hit the network — it returns a + // canned CliRunDiff so `--dry-run` shows the shape offline. + const diff = await runDiff( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runA: 'run_aaa', + runB: 'run_bbb', + }, + { stdout: () => undefined, stderr: () => undefined }, + ); + expect(diff.runA.runId).toBe('run_aaa'); + expect(diff.runB.runId).toBe('run_bbb'); + expect(diff.verdictChanged).toBe(true); + expect(diff.changedSteps).toHaveLength(1); + expect(diff.changedSteps[0]).toMatchObject({ + stepIndex: 2, + statusA: 'passed', + statusB: 'failed', + }); + }); +}); + describe('runResult', () => { it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index ede6376..7fe985d 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3664,6 +3664,196 @@ function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestSte }; } +export interface DiffOptions extends CommonOptions { + runA: string; + runB: string; +} + +/** One step whose status flipped between the two compared runs. */ +export interface CliDiffStep { + stepIndex: number; + statusA: string; + statusB: string; + /** First divergent failing side's error text, when the wire carried one. */ + errorA?: string | null; + errorB?: string | null; +} + +export interface CliRunDiff { + runA: { + runId: string; + testId: string; + status: string; + failureKind: string | null; + failedStepIndex: number | null; + codeVersion: string | null; + }; + runB: { + runId: string; + testId: string; + status: string; + failureKind: string | null; + failedStepIndex: number | null; + codeVersion: string | null; + }; + verdictChanged: boolean; + failedStepIndexChanged: boolean; + failureKindChanged: boolean; + codeVersionChanged: boolean; + /** True when the two runs belong to DIFFERENT tests (deltas may be meaningless). */ + crossTest: boolean; + changedSteps: CliDiffStep[]; +} + +/** + * `test diff ` (issue #124): isolate what regressed between two + * runs, the first question when CI goes red ("what changed since the last + * green run?"). Pure client-side composition of the existing per-run read + * (`GET /runs/{id}?includeSteps=true`); the endpoint accepts any two run-ids, + * so a cross-test pair is a WARNING, not an error. Exit 0 when the verdicts + * match, exit 1 when they differ, so the command is CI-scriptable. + */ +export async function runDiff(opts: DiffOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const sample: CliRunDiff = { + runA: { + runId: opts.runA, + testId: 'test_dryrun', + status: 'passed', + failureKind: null, + failedStepIndex: null, + codeVersion: 'v1', + }, + runB: { + runId: opts.runB, + testId: 'test_dryrun', + status: 'failed', + failureKind: 'assertion', + failedStepIndex: 2, + codeVersion: 'v1', + }, + verdictChanged: true, + failedStepIndexChanged: true, + failureKindChanged: true, + codeVersionChanged: false, + crossTest: false, + changedSteps: [{ stepIndex: 2, statusA: 'passed', statusB: 'failed' }], + }; + out.print(sample, () => renderRunDiffText(sample)); + return sample; + } + + const client = makeClient(opts, deps); + const [runA, runB] = await Promise.all([ + client.getRun(opts.runA, { includeSteps: true }), + client.getRun(opts.runB, { includeSteps: true }), + ]); + + const crossTest = runA.testId !== runB.testId; + if (crossTest) { + stderrFn( + `⚠ the two runs belong to different tests (${runA.testId} vs ${runB.testId}) — step deltas may be meaningless`, + ); + } + + const stepsByIndex = ( + run: RunResponse, + ): Map => { + const map = new Map(); + for (const step of run.steps ?? []) { + const index = parseInt(step.stepIndex, 10); + if (Number.isInteger(index)) + map.set(index, { status: step.status ?? 'unknown', error: step.error }); + } + return map; + }; + const stepsA = stepsByIndex(runA); + const stepsB = stepsByIndex(runB); + const allIndexes = [...new Set([...stepsA.keys(), ...stepsB.keys()])].sort( + (left, right) => left - right, + ); + const changedSteps: CliDiffStep[] = []; + for (const index of allIndexes) { + const sideA = stepsA.get(index); + const sideB = stepsB.get(index); + const statusA = sideA?.status ?? 'absent'; + const statusB = sideB?.status ?? 'absent'; + if (statusA === statusB) continue; + changedSteps.push({ + stepIndex: index, + statusA, + statusB, + ...(sideA?.error ? { errorA: sideA.error } : {}), + ...(sideB?.error ? { errorB: sideB.error } : {}), + }); + } + + const summarize = (run: RunResponse) => ({ + runId: run.runId, + testId: run.testId, + status: run.status, + failureKind: run.failureKind ?? null, + failedStepIndex: run.failedStepIndex, + codeVersion: run.codeVersion ?? null, + }); + const diff: CliRunDiff = { + runA: summarize(runA), + runB: summarize(runB), + verdictChanged: runA.status !== runB.status, + failedStepIndexChanged: runA.failedStepIndex !== runB.failedStepIndex, + failureKindChanged: (runA.failureKind ?? null) !== (runB.failureKind ?? null), + codeVersionChanged: (runA.codeVersion ?? null) !== (runB.codeVersion ?? null), + crossTest, + changedSteps, + }; + out.print(diff, () => renderRunDiffText(diff)); + + if (diff.verdictChanged) { + // Result already printed; the typed exit makes `test diff` a CI gate. + throw new CLIError( + `verdicts differ: ${runA.runId}=${runA.status} vs ${runB.runId}=${runB.status}`, + 1, + ); + } + return diff; +} + +function renderRunDiffText(diff: CliRunDiff): string { + const lines: string[] = []; + lines.push(`runA: ${diff.runA.runId} ${diff.runA.status} (test ${diff.runA.testId})`); + lines.push(`runB: ${diff.runB.runId} ${diff.runB.status} (test ${diff.runB.testId})`); + lines.push( + `verdict: ${diff.verdictChanged ? `${diff.runA.status} -> ${diff.runB.status}` : `unchanged (${diff.runA.status})`}`, + ); + if (diff.failureKindChanged) + lines.push( + `failureKind: ${diff.runA.failureKind ?? '(none)'} -> ${diff.runB.failureKind ?? '(none)'}`, + ); + if (diff.failedStepIndexChanged) + lines.push( + `failedStepIndex: ${diff.runA.failedStepIndex ?? '(none)'} -> ${diff.runB.failedStepIndex ?? '(none)'}`, + ); + lines.push( + `codeVersion: ${diff.codeVersionChanged ? `${diff.runA.codeVersion ?? '(none)'} -> ${diff.runB.codeVersion ?? '(none)'} (code drift)` : 'unchanged'}`, + ); + if (diff.changedSteps.length === 0) { + lines.push('steps: no per-step status changes'); + } else { + lines.push(`steps changed: ${diff.changedSteps.length}`); + for (const step of diff.changedSteps) { + lines.push(` #${step.stepIndex} ${step.statusA} -> ${step.statusB}`); + if (step.errorB) lines.push(` error(B): ${step.errorB.replace(/\s+/g, ' ').trim()}`); + else if (step.errorA) + lines.push(` error(A): ${step.errorA.replace(/\s+/g, ' ').trim()}`); + } + } + return lines.join('\n'); +} + export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -7413,6 +7603,16 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('diff ') + .description( + 'Compare two runs and print what regressed: verdict, failureKind, failedStepIndex, per-step status flips, codeVersion drift. Exit 0 when verdicts match, 1 when they differ.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (runA: string, runB: string, _cmdOpts: unknown, command: Command) => { + await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps); + }); + test .command('result ') .description( diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 03cb551..f8e9fa2 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -200,6 +200,11 @@ Commands: steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) + diff Compare two runs and print what + regressed: verdict, failureKind, + failedStepIndex, per-step status flips, + codeVersion drift. Exit 0 when verdicts + match, 1 when they differ. result [options] Get the latest result for a test (default) or list prior runs (--history). --output json shape differs by mode: From 011208c46f3d132e98cb3138b0b36770eb05e639 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:34:11 -0400 Subject: [PATCH 040/117] feat(agent): stamp installed skills with a version/hash marker and add "agent status" (#177) --- src/commands/agent.test.ts | 129 ++++++++- src/commands/agent.ts | 265 +++++++++++++++++- src/lib/agent-targets.test.ts | 113 ++++++++ src/lib/agent-targets.ts | 146 +++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 3 + 5 files changed, 645 insertions(+), 11 deletions(-) diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 493cc4e..f8ed59f 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -9,13 +9,21 @@ import { MANAGED_SECTION_END, ONBOARD_CODEX_LINE, SKILLS, + buildSkillMarker, pathFor, renderForTarget, + renderOwnFileWithMarker, TARGETS, type AgentTarget, } from '../lib/agent-targets.js'; -import type { AgentDeps, AgentFs, InstallResult, ListResult } from './agent.js'; -import { AGENTS_MD_CODEX_BUDGET_BYTES, createAgentCommand, runInstall, runList } from './agent.js'; +import type { AgentDeps, AgentFs, InstallResult, ListResult, StatusResult } from './agent.js'; +import { + AGENTS_MD_CODEX_BUDGET_BYTES, + createAgentCommand, + runInstall, + runList, + runStatus, +} from './agent.js'; // --------------------------------------------------------------------------- // In-memory AgentFs backed by a Map @@ -2430,3 +2438,120 @@ describe('runInstall — SKILLS registry / DEFAULT_SKILLS contract', () => { expect(ONBOARD_CODEX_LINE).toContain('**First-time setup:**'); }); }); + +// --------------------------------------------------------------------------- +// runStatus — `agent status` (issue #123) +// --------------------------------------------------------------------------- + +describe('runStatus — agent status (issue #123)', () => { + const statusOpts = { + profile: 'default' as const, + output: 'json' as const, + debug: false, + dryRun: false, + }; + + /** Run status against the given fs and return the printed rows. */ + async function statusRows(agentFs: AgentFs): Promise<{ rows: StatusResult[]; thrown: unknown }> { + const { capture, deps } = makeCapture(); + let thrown: unknown; + try { + await runStatus(statusOpts, { cwd: CWD, fs: agentFs, ...deps }); + } catch (err) { + thrown = err; + } + return { rows: JSON.parse(capture.stdout.join('')) as StatusResult[], thrown }; + } + + it('nothing installed: every row is absent and the command exits 0', async () => { + const { fs: agentFs } = makeMemFs(); + const { rows, thrown } = await statusRows(agentFs); + expect(thrown).toBeUndefined(); + expect(rows).toHaveLength(Object.keys(TARGETS).length * DEFAULT_SKILLS.length); + expect(rows.every(row => row.state === 'absent')).toBe(true); + }); + + it('fresh installs read ok (own-file and codex managed section), exit 0', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'codex'], + skills: [...DEFAULT_SKILLS], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(thrown).toBeUndefined(); + for (const skill of DEFAULT_SKILLS) { + expect(rows.find(r => r.target === 'claude' && r.skill === skill)?.state).toBe('ok'); + expect(rows.find(r => r.target === 'codex' && r.skill === skill)?.state).toBe('ok'); + expect(rows.find(r => r.target === 'cursor' && r.skill === skill)?.state).toBe('absent'); + } + }); + + it('stale: a marker whose hash matches an OLDER body reads stale and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const oldBody = '# TestSprite Verification Loop\n\nold body from a previous CLI release\n'; + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + renderOwnFileWithMarker( + 'claude', + 'testsprite-verify', + buildSkillMarker('testsprite-verify', oldBody), + oldBody, + ), + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'stale', + ); + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(1); + expect((thrown as CLIError).message).toContain('need attention'); + }); + + it('modified: current hash but edited bytes reads modified and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const canonical = renderForTarget('claude', 'testsprite-verify').content; + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + `${canonical}\n\n`, + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'modified', + ); + expect((thrown as CLIError).exitCode).toBe(1); + }); + + it('unmarked: an artifact without a marker line reads unmarked and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + '# hand-rolled skill file with no marker\n', + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'unmarked', + ); + expect((thrown as CLIError).exitCode).toBe(1); + }); + + it('rejects an explicit empty --dir (exit 5), matching the resolve-to-cwd hazard', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + await expect( + runStatus({ ...statusOpts, dir: ' ' }, { cwd: CWD, fs: agentFs, ...deps }), + ).rejects.toMatchObject({ exitCode: 5 }); + }); +}); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 8213755..1b4e878 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -11,10 +11,15 @@ import { TARGETS, SKILLS, DEFAULT_SKILLS, + MARKER_SKILL_SEPARATOR, pathFor, loadSkillBodyFor, + bodyHash12, buildCodexAggregate, + buildSkillMarker, + parseSkillMarker, renderForTarget, + renderOwnFileWithMarker, MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, } from '../lib/agent-targets.js'; @@ -150,11 +155,15 @@ async function writeBackup(agentFs: AgentFs, abs: string, existing: string): Pro // --------------------------------------------------------------------------- /** - * Build the section block to inject (sentinels + body + trailing newline). + * Build the section block to inject (sentinels + marker + body + trailing + * newline). The provenance marker line sits just inside the BEGIN sentinel so + * `agent status` can fingerprint the section. The same skill set + CLI version + * + body always produce byte-identical output, so the classifySection + * 'unchanged' fast-path keeps working across re-installs. * Uses \n throughout; the caller handles CRLF normalisation. */ -function buildSection(body: string): string { - return `${MANAGED_SECTION_BEGIN}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; +function buildSection(body: string, markerLine: string): string { + return `${MANAGED_SECTION_BEGIN}\n${markerLine}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; } /** @@ -437,7 +446,14 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr let codexSectionCache: string | undefined; const getCodexSection = (): string => { if (codexSectionCache === undefined) { - codexSectionCache = buildSection(buildCodexAggregate(skills)); + const aggregate = buildCodexAggregate(skills); + // ONE marker for the whole managed section: it names every aggregated + // skill ('+'-joined) and hashes the canonical aggregate body, so + // `agent status` can attribute and fingerprint the section per skill. + codexSectionCache = buildSection( + aggregate, + buildSkillMarker(skills.join(MARKER_SKILL_SEPARATOR), aggregate), + ); } return codexSectionCache; }; @@ -789,6 +805,236 @@ export async function runList(opts: CommonOptions, deps: AgentDeps = {}): Promis }); } +// --------------------------------------------------------------------------- +// runStatus (issue #123: detect silently stale installed skill files) +// --------------------------------------------------------------------------- + +/** + * Health of one installed skill artifact, as reported by `agent status`. + * + * Decision order (first match wins): + * - 'absent' : nothing at the landing path (codex: no managed section, + * including an AGENTS.md that exists without our sentinels). + * - 'corrupt' : codex only. Dangling or duplicated sentinels, the same + * classification `agent install` refuses on; status REPORTS it + * instead of refusing. + * - 'unmarked' : artifact present but carries no testsprite-skill marker + * (installed before markers existed), or the landing path is + * occupied by a non-regular file (never followed). + * - 'stale' : marker present, but its hash differs from the current + * canonical body: a re-install would change the content. Edits + * on top of an OLD install also read stale (older renders + * cannot be reproduced); the remedy is the same re-install. + * - 'modified' : marker hash matches the current body, but the artifact bytes + * differ from the canonical render carrying that same marker + * line: the user edited the artifact after install. + * - 'ok' : marker hash matches and the bytes equal the canonical render + * with the file's own marker line (a version-string-only lag + * with an unchanged body still reads ok). + * + * For the codex managed section, ONE marker names every aggregated skill + * ('+'-joined); skills not named by the marker report 'absent'. + */ +export type SkillArtifactState = 'ok' | 'stale' | 'modified' | 'unmarked' | 'absent' | 'corrupt'; + +export interface StatusResult { + target: AgentTarget; + skill: string; + path: string; + state: SkillArtifactState; +} + +interface StatusOptions extends CommonOptions { + dir?: string; +} + +/** + * Classify one own-file artifact per the {@link SkillArtifactState} contract. + * Comparisons are byte-exact, matching the installer's own skipped/blocked + * comparison for own-file targets. + */ +async function classifyOwnFileState( + agentFs: AgentFs, + abs: string, + target: AgentTarget, + skill: string, + bodyForSkill: (skill: string) => string, +): Promise { + const stat = await agentFs.lstat(abs); + if (stat === null) return 'absent'; + // Occupied by a directory or symlink: not something our installer wrote, and + // never followed (mirrors the installer's fail-closed stance on symlinks). + if (!stat.isFile) return 'unmarked'; + + const existing = await agentFs.readFile(abs); + const marker = parseSkillMarker(existing); + if (marker === null) return 'unmarked'; + + const canonicalBody = bodyForSkill(skill); + if (marker.hash12 !== bodyHash12(canonicalBody)) return 'stale'; + + // Hash matches the current body: pristine iff the file equals the canonical + // render carrying its own marker line, so a marker whose version string lags + // behind an unchanged body still reads ok. + const reRender = renderOwnFileWithMarker(target, skill, marker.line, canonicalBody); + return existing === reRender ? 'ok' : 'modified'; +} + +/** + * Classify the codex managed section per skill. The section is ONE artifact + * carrying ONE marker that names every aggregated skill, so a single + * inspection answers all skill rows; the returned function maps a skill name + * to its state. Comparisons are CRLF-insensitive on the section bytes. + */ +async function classifyManagedSectionStates( + agentFs: AgentFs, + abs: string, +): Promise<(skill: string) => SkillArtifactState> { + const constantState = + (state: SkillArtifactState): ((skill: string) => SkillArtifactState) => + () => + state; + + const stat = await agentFs.lstat(abs); + if (stat === null) return constantState('absent'); + // Occupied by a directory or symlink: never followed (fail-closed). + if (!stat.isFile) return constantState('unmarked'); + + const existing = await agentFs.readFile(abs); + + // Current canonical section for the default skill set. classifySection's + // 'unchanged' answers the common all-defaults-fresh case; its + // corrupt/append classification is reused verbatim for status verdicts. + const defaultAggregate = buildCodexAggregate(DEFAULT_SKILLS); + const defaultSection = buildSection( + defaultAggregate, + buildSkillMarker(DEFAULT_SKILLS.join(MARKER_SKILL_SEPARATOR), defaultAggregate), + ); + const sectionState = classifySection(existing, defaultSection); + + if (sectionState.kind === 'corrupt') return constantState('corrupt'); + // No standalone sentinels anywhere: the managed section is not installed. + if (sectionState.kind === 'append') return constantState('absent'); + if (sectionState.kind === 'unchanged') { + // Byte-identical to today's default install. + return skill => ((DEFAULT_SKILLS as readonly string[]).includes(skill) ? 'ok' : 'absent'); + } + if (sectionState.kind !== 'replace') { + // 'create' is unreachable when the file exists; treat defensively as absent. + return constantState('absent'); + } + + // Sentinels are present but the section differs from today's default + // canonical: slice the live section bytes out of the file and inspect its + // own marker (before/after are exact byte prefix/suffix around the section). + const sectionContent = existing.slice( + sectionState.before.length, + existing.length - sectionState.after.length, + ); + const marker = parseSkillMarker(sectionContent); + if (marker === null) return constantState('unmarked'); + + const installedSkills = marker.skill.split(MARKER_SKILL_SEPARATOR); + const coversSkill = (skill: string): boolean => installedSkills.includes(skill); + + // A marker naming a skill this CLI does not ship cannot be re-rendered; + // report the named skills stale (a re-install refreshes the section). + if (installedSkills.some(name => SKILLS[name] === undefined)) { + return skill => (coversSkill(skill) ? 'stale' : 'absent'); + } + + const canonicalAggregate = buildCodexAggregate(installedSkills); + if (marker.hash12 !== bodyHash12(canonicalAggregate)) { + return skill => (coversSkill(skill) ? 'stale' : 'absent'); + } + + // Hash matches the current aggregate: the section is pristine iff its bytes + // equal a re-render carrying its own marker line (version-string-only lag + // with an unchanged body still reads ok). + const pristine = + sectionContent.replace(/\r\n/g, '\n') === buildSection(canonicalAggregate, marker.line); + return skill => (coversSkill(skill) ? (pristine ? 'ok' : 'modified') : 'absent'); +} + +/** + * `agent status`: one row per (target × default skill), each classified per + * the {@link SkillArtifactState} contract. Exit contract: returns normally + * (exit 0) when every row is 'ok' or 'absent'; throws CLIError exit 1 when any + * row is stale/modified/unmarked/corrupt, so the command can gate CI. + */ +export async function runStatus(opts: StatusOptions, deps: AgentDeps = {}): Promise { + const agentFs = deps.fs ?? defaultAgentFs; + const out = makeOutput(opts.output, deps); + + // An explicit but empty --dir must not silently resolve to cwd + // (path.resolve('') === cwd). + if (opts.dir !== undefined && opts.dir.trim() === '') { + throw localValidationError('dir', 'must not be empty'); + } + const dir = opts.dir !== undefined ? opts.dir.trim() : (deps.cwd ?? process.cwd()); + const root = path.resolve(dir); + + // Canonical own-file bodies, read once per skill (same lazy caching pattern + // as runInstall's bodyForSkill). + const skillBodyCache = new Map(); + const bodyForSkill = (skill: string): string => { + let cachedBody = skillBodyCache.get(skill); + if (cachedBody === undefined) { + cachedBody = loadSkillBodyFor(skill); + skillBodyCache.set(skill, cachedBody); + } + return cachedBody; + }; + + const results: StatusResult[] = []; + for (const [target, spec] of Object.entries(TARGETS) as [ + AgentTarget, + { mode: string; path: string }, + ][]) { + if (spec.mode === 'managed-section') { + const stateFor = await classifyManagedSectionStates(agentFs, path.resolve(root, spec.path)); + for (const skill of DEFAULT_SKILLS) { + results.push({ target, skill, path: spec.path, state: stateFor(skill) }); + } + continue; + } + for (const skill of DEFAULT_SKILLS) { + const relPath = pathFor(target, skill); + results.push({ + target, + skill, + path: relPath, + state: await classifyOwnFileState( + agentFs, + path.resolve(root, relPath), + target, + skill, + bodyForSkill, + ), + }); + } + } + + out.print(results, data => { + const items = data as StatusResult[]; + const header = `${'TARGET'.padEnd(14)} ${'SKILL'.padEnd(20)} ${'STATE'.padEnd(10)} PATH`; + const rows = items.map( + row => `${row.target.padEnd(14)} ${row.skill.padEnd(20)} ${row.state.padEnd(10)} ${row.path}`, + ); + return [header, ...rows].join('\n'); + }); + + const needingAttention = results.filter( + result => result.state !== 'ok' && result.state !== 'absent', + ); + if (needingAttention.length > 0) { + throw new CLIError( + `${needingAttention.length} skill artifact(s) need attention (stale/modified/unmarked/corrupt); re-run \`testsprite agent install\` (add --force for own-file targets) to refresh them.`, + 1, + ); + } +} + // --------------------------------------------------------------------------- // Command factory // --------------------------------------------------------------------------- @@ -852,6 +1098,17 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { await runList(resolveCommonOptions(command), deps); }); + agent + .command('status') + .description( + 'Check installed TestSprite skill files against this CLI version: ok, stale, modified, unmarked, absent, or corrupt (exits 1 when anything needs attention, so it can gate CI)', + ) + .option('--dir ', 'Project root to inspect (default: cwd)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: { dir?: string }, command: Command) => { + await runStatus({ ...resolveCommonOptions(command), dir: cmdOpts.dir }, deps); + }); + return agent; } diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 8ef0e36..38c74e2 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -1,5 +1,7 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; +import { VERSION } from '../version.js'; import { DEFAULT_SKILLS, MANAGED_SECTION_BEGIN, @@ -9,13 +11,17 @@ import { SKILL_NAME, SKILLS, TARGETS, + bodyHash12, buildCodexAggregate, + buildSkillMarker, codexContentFor, loadCodexSkillBody, loadSkillBody, loadSkillBodyFor, + parseSkillMarker, pathFor, renderForTarget, + renderOwnFileWithMarker, } from './agent-targets.js'; // --------------------------------------------------------------------------- @@ -699,3 +705,110 @@ describe('renderForTarget for testsprite-onboard', () => { expect(() => renderForTarget('claude', 'testsprite-unknown')).toThrow('unknown skill'); }); }); + +// --------------------------------------------------------------------------- +// Install marker (issue #123): format, parsing, and render placement +// --------------------------------------------------------------------------- + +describe('buildSkillMarker / parseSkillMarker / bodyHash12', () => { + it('marker line is an HTML comment carrying name, vVERSION, and a 12-hex hash', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + expect(marker).toBe( + ``, + ); + expect(marker).toMatch( + /^$/, + ); + }); + + it('bodyHash12 equals the first 12 hex chars of the body SHA-256', () => { + const fullHex = createHash('sha256').update(STUB_BODY, 'utf8').digest('hex'); + expect(bodyHash12(STUB_BODY)).toBe(fullHex.slice(0, 12)); + }); + + it('parseSkillMarker round-trips a built marker embedded in surrounding content', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + const parsed = parseSkillMarker(`# heading\n${marker}\nbody text\n`); + expect(parsed).not.toBeNull(); + expect(parsed?.skill).toBe('testsprite-verify'); + expect(parsed?.version).toBe(VERSION); + expect(parsed?.hash12).toBe(bodyHash12(STUB_BODY)); + expect(parsed?.line).toBe(marker); + }); + + it('parseSkillMarker strips a trailing CR so CRLF checkouts parse identically', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + const parsed = parseSkillMarker(`${marker}\r\nrest\r\n`); + expect(parsed?.line).toBe(marker); + }); + + it('parseSkillMarker returns null when no marker line is present', () => { + expect(parseSkillMarker('# Just a heading\n\nProse without any marker.\n')).toBeNull(); + }); + + it('parseSkillMarker ignores the managed-section sentinels (also HTML comments)', () => { + expect(parseSkillMarker(`${MANAGED_SECTION_BEGIN}\nbody\n${MANAGED_SECTION_END}\n`)).toBeNull(); + }); +}); + +describe('render marker placement (own-file targets)', () => { + it('claude render carries the marker on the line right after the closing frontmatter fence', () => { + const { content } = renderForTarget('claude', 'testsprite-verify', STUB_BODY); + const closingFence = '\n---\n'; + const fenceEnd = content.indexOf(closingFence) + closingFence.length; + expect(content.slice(fenceEnd).startsWith(''; + const withForeign = renderOwnFileWithMarker( + 'claude', + 'testsprite-verify', + foreignMarker, + STUB_BODY, + ); + expect(withForeign).toContain(foreignMarker); + // Marker line aside, the bytes match the canonical render exactly. + const canonical = renderForTarget('claude', 'testsprite-verify', STUB_BODY).content; + const currentMarker = buildSkillMarker('testsprite-verify', STUB_BODY); + expect(withForeign.replace(foreignMarker, currentMarker)).toBe(canonical); + }); + + it('renderOwnFileWithMarker rejects the managed-section target', () => { + expect(() => renderOwnFileWithMarker('codex', 'testsprite-verify', 'marker', 'body')).toThrow( + 'own-file', + ); + }); + + it('renderOwnFileWithMarker throws on an unknown skill', () => { + expect(() => renderOwnFileWithMarker('claude', 'testsprite-unknown', 'marker')).toThrow( + 'unknown skill', + ); + }); +}); diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 663a998..0273a04 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -1,4 +1,6 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; +import { VERSION } from '../version.js'; export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex' | 'kiro'; @@ -211,6 +213,85 @@ export const MANAGED_SECTION_BEGIN = ''; export const MANAGED_SECTION_END = ''; +// --------------------------------------------------------------------------- +// Install marker (stale-skill detection, issue #123) +// --------------------------------------------------------------------------- + +/** + * Hex characters of the canonical body's SHA-256 kept in the install marker. + * 12 hex chars (48 bits) is ample for drift DETECTION (equality against bodies + * this CLI ships); the marker is provenance metadata, not a security boundary. + */ +const MARKER_HASH_HEX_LENGTH = 12; + +/** + * When one marker covers several skills (the codex managed section aggregates + * every installed skill), their names are joined with this separator in the + * marker's skill field. Skill names never contain '+' (see {@link SKILLS} keys). + */ +export const MARKER_SKILL_SEPARATOR = '+'; + +/** + * Marker line shape: ``. + * An HTML comment is inert in every target format (SKILL.md, .mdc, .clinerules + * markdown, AGENTS.md). Built via `new RegExp` so the hash length stays bound + * to {@link MARKER_HASH_HEX_LENGTH}. + */ +const SKILL_MARKER_LINE_RE = new RegExp( + `^$`, +); + +/** + * First {@link MARKER_HASH_HEX_LENGTH} hex chars of the SHA-256 of a canonical + * skill body. The hash covers the CANONICAL BODY ONLY (pre-wrap, pre-marker), + * so writing the marker into the rendered artifact never changes the hash the + * marker itself carries. + */ +export function bodyHash12(canonicalBody: string): string { + return createHash('sha256') + .update(canonicalBody, 'utf8') + .digest('hex') + .slice(0, MARKER_HASH_HEX_LENGTH); +} + +/** + * Build the provenance marker line for a skill (or a + * {@link MARKER_SKILL_SEPARATOR}-joined skill set) and its canonical body. + * `agent status` compares this fingerprint against the bodies the running CLI + * ships to detect silently stale installs. + */ +export function buildSkillMarker(skillName: string, canonicalBody: string): string { + return ``; +} + +/** A marker line parsed back into its fields. */ +export interface ParsedSkillMarker { + /** Skill name, or several names joined with {@link MARKER_SKILL_SEPARATOR}. */ + skill: string; + /** CLI version that wrote the artifact. */ + version: string; + /** First 12 hex chars of the canonical body's SHA-256 at install time. */ + hash12: string; + /** The exact marker line (trailing CR/whitespace stripped) as found. */ + line: string; +} + +/** + * Find the first testsprite-skill marker line in `content`, or null when the + * content carries none (a pre-marker install). Lines are matched whole with + * trailing CR/whitespace stripped, so CRLF checkouts parse identically. + */ +export function parseSkillMarker(content: string): ParsedSkillMarker | null { + for (const rawLine of content.split('\n')) { + const line = rawLine.trimEnd(); + const matched = SKILL_MARKER_LINE_RE.exec(line); + if (matched) { + return { skill: matched[1]!, version: matched[2]!, hash12: matched[3]!, line }; + } + } + return null; +} + type ReadFn = (url: URL) => string; const defaultRead: ReadFn = (url: URL) => readFileSync(url, 'utf8'); @@ -285,15 +366,67 @@ export function loadCodexSkillBody(read: ReadFn = defaultRead): string { // renderForTarget // --------------------------------------------------------------------------- +/** + * Place the marker line inside a wrapped own-file render. + * + * - Wraps that emit YAML frontmatter (claude/antigravity/cursor): the marker + * lands on the line right after the closing `---` fence, before the body. + * - Wrapless targets (cline, body verbatim): the marker is appended as the + * LAST line instead. Cline surfaces the file's first heading as the rule + * title, so a leading comment would displace the body's H1. + */ +function injectMarkerLine(wrapped: string, markerLine: string): string { + if (wrapped.startsWith('---\n')) { + // The name/description frontmatter values are single-line, so the first + // `\n---\n` after the opening fence is always the closing fence. + const closingFence = '\n---\n'; + const fenceIdx = wrapped.indexOf(closingFence); + if (fenceIdx !== -1) { + const insertAt = fenceIdx + closingFence.length; + return `${wrapped.slice(0, insertAt)}${markerLine}\n${wrapped.slice(insertAt)}`; + } + } + const separator = wrapped.endsWith('\n') ? '' : '\n'; + return `${wrapped}${separator}${markerLine}\n`; +} + +/** + * Exact own-file bytes for a skill on a target, carrying the GIVEN marker line. + * `agent status` uses this to re-render the current canonical body with a + * file's own (possibly older-versioned) marker: when only the marker's version + * string lags but the body is unchanged, the artifact still compares pristine. + */ +export function renderOwnFileWithMarker( + target: AgentTarget, + skill: string, + markerLine: string, + body?: string, +): string { + const spec = TARGETS[target]; + if (spec.mode !== 'own-file') { + throw new Error(`renderOwnFileWithMarker: ${target} is not an own-file target`); + } + const skillSpec = SKILLS[skill]; + if (!skillSpec) throw new Error(`unknown skill: ${skill}`); + const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); + return injectMarkerLine( + spec.wrap(skillSpec.name, skillSpec.description, resolvedBody), + markerLine, + ); +} + /** * The exact bytes to write for one skill on one target. * * - own-file targets: `body` defaults to the skill's own-file asset, wrapped in - * the target's frontmatter/header. + * the target's frontmatter/header, and carrying a provenance marker line so + * `agent status` can tell fresh, stale, and hand-edited installs apart. * - codex (managed-section): returns the skill's codex contribution unwrapped - * (plain Markdown, no frontmatter). The real install does NOT call this for - * codex — it aggregates all skills via {@link buildCodexAggregate} — but it is - * kept single-skill here for tests and parity. Pass an explicit `body` to override. + * and marker-free (plain Markdown, no frontmatter). The real install does NOT + * call this for codex: it aggregates all skills via + * {@link buildCodexAggregate} and writes ONE marker just inside the BEGIN + * sentinel. It is kept single-skill here for tests and parity. Pass an + * explicit `body` to override. */ export function renderForTarget( t: AgentTarget, @@ -309,5 +442,8 @@ export function renderForTarget( return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; } const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); - return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; + return { + path, + content: renderOwnFileWithMarker(t, skill, buildSkillMarker(skill, resolvedBody), resolvedBody), + }; } diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index f8e9fa2..0c387fa 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -14,6 +14,9 @@ Commands: first-run onboarding) into a project for a coding agent list List supported agent targets and skills, their status, and landing paths + status [options] Check installed TestSprite skill files against this CLI + version: ok, stale, modified, unmarked, absent, or corrupt + (exits 1 when anything needs attention, so it can gate CI) help [command] display help for command " `; From f732fdb1663b4dddb3a75db6e471359fed6fa054 Mon Sep 17 00:00:00 2001 From: kshitij-heizen Date: Mon, 6 Jul 2026 01:04:26 +0530 Subject: [PATCH 041/117] fix(bundle): only sweep bundle-owned files in commit, preserving user files in --out dir (#162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commitBundle's stale-file sweep removed EVERY directory entry not part of the fresh bundle, so 'test failure get --out

' / 'test artifact get --out ' pointed at a pre-existing, populated directory silently deleted the user's unrelated files (exit 0, no warning) — on the very first write, not just re-commits. Scope the sweep to entries the bundle format owns (result.json, failure.json, video.mp4, meta.json, steps, .tmp, .partial, code.). Stale bundle files are still cleaned — an old video.mp4 when the new bundle has no video, a code.py when the new bundle writes code.ts — but foreign files and directories are never touched. Fixes #159 Co-authored-by: Kshitij Bhardwaj Co-authored-by: Claude Fable 5 --- src/lib/bundle.test.ts | 77 +++++++++++++++++++++++++++++++++++++++++- src/lib/bundle.ts | 36 +++++++++++++++++--- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index 5b746da..0c0680c 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -8,7 +8,7 @@ * the full http+fetch path is wired against MSW). */ -import { existsSync, mkdtempSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -18,6 +18,7 @@ import { assertNoEscape, BUNDLE_SCHEMA_VERSION, buildMeta, + isBundleOwnedEntry, pickCodeExtension, resolveBundleDir, STREAM_URL_MAX_RETRIES, @@ -694,6 +695,37 @@ describe('streamUrlToFile retry', () => { }); }); +describe('isBundleOwnedEntry', () => { + it('owns the fixed bundle file set', () => { + for (const entry of [ + 'result.json', + 'failure.json', + 'video.mp4', + 'meta.json', + 'steps', + '.tmp', + '.partial', + ]) { + expect(isBundleOwnedEntry(entry)).toBe(true); + } + }); + + it('owns code. for any single-token extension', () => { + expect(isBundleOwnedEntry('code.ts')).toBe(true); + expect(isBundleOwnedEntry('code.js')).toBe(true); + expect(isBundleOwnedEntry('code.py')).toBe(true); + }); + + it('does not own foreign entries', () => { + expect(isBundleOwnedEntry('notes.txt')).toBe(false); + expect(isBundleOwnedEntry('src')).toBe(false); + expect(isBundleOwnedEntry('.git')).toBe(false); + expect(isBundleOwnedEntry('code.tar.gz')).toBe(false); + expect(isBundleOwnedEntry('mycode.ts')).toBe(false); + expect(isBundleOwnedEntry('code.')).toBe(false); + }); +}); + describe('step artifact path validation', () => { // A fetchImpl that fails the test if called — proves validation rejects // before any write happens. @@ -807,6 +839,49 @@ describe('step artifact path validation', () => { expect(existsSync(join(res.dir, 'meta.json'))).toBe(true); }); + describe('commit sweep ownership (data-loss guard)', () => { + it('preserves pre-existing foreign files and directories in the --out dir', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'notes.txt'), 'important notes\n', 'utf8'); + mkdirSync(join(dir, 'src')); + writeFileSync(join(dir, 'src', 'app.js'), "console.log('app')\n", 'utf8'); + + const res = await writeBundle(stepCtx(3), { + dir, + failedOnly: false, + fetchImpl: throwIfFetched, + }); + + // The bundle landed… + expect(existsSync(join(res.dir, 'meta.json'))).toBe(true); + expect(existsSync(join(res.dir, 'result.json'))).toBe(true); + // …and the user's unrelated files survived the commit sweep. + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('important notes\n'); + expect(readFileSync(join(dir, 'src', 'app.js'), 'utf8')).toBe("console.log('app')\n"); + }); + + it('still sweeps a stale bundle-owned video.mp4 the new bundle does not write', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'video.mp4'), 'stale-bytes', 'utf8'); + + // stepCtx has videoUrl: null → the fresh bundle ships no video. + await writeBundle(stepCtx(3), { dir, failedOnly: false, fetchImpl: throwIfFetched }); + + expect(existsSync(join(dir, 'video.mp4'))).toBe(false); + }); + + it('sweeps a stale code file with a different extension than the new bundle writes', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'code.py'), '# stale python code\n', 'utf8'); + + // baseCtx.code.language is 'typescript' → the fresh bundle writes code.ts. + await writeBundle(stepCtx(3), { dir, failedOnly: false, fetchImpl: throwIfFetched }); + + expect(existsSync(join(dir, 'code.ts'))).toBe(true); + expect(existsSync(join(dir, 'code.py'))).toBe(false); + }); + }); + describe('assertNoEscape', () => { it('returns the resolved path for an in-bounds segment', () => { const base = mkdtempSync(join(tmpdir(), 'bundle-test-')); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index 7795115..9c03b07 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -543,6 +543,30 @@ async function freshTmpDir(dir: string): Promise { * caught reading the dir during it sees no meta and refuses to consume * (per §7.3). That's what we want. */ +/** + * Whether a top-level directory entry belongs to the bundle format — + * i.e. something a prior `writeBundle` could have produced and this + * commit is therefore allowed to clean up. `code.` is matched by + * pattern (not the current run's extension) so a stale `code.py` is + * still swept when the new bundle writes `code.ts`. Everything else in + * the directory is the user's and must never be deleted (`--out` can + * point at a pre-existing, populated directory). + */ +export function isBundleOwnedEntry(entry: string): boolean { + if ( + entry === 'result.json' || + entry === 'failure.json' || + entry === 'video.mp4' || + entry === 'meta.json' || + entry === 'steps' || + entry === '.tmp' || + entry === '.partial' + ) { + return true; + } + return /^code\.[A-Za-z0-9]+$/.test(entry); +} + async function commitBundle( tmpDir: string, dir: string, @@ -553,20 +577,22 @@ async function commitBundle( // (2) Sweep stale top-level files that the new bundle won't write. // If the prior run wrote `video.mp4` and the new run has no video, - // an in-place rename leaves the old video lingering. Enumerate - // current top-level entries and remove anything that isn't being - // freshly renamed in. + // an in-place rename leaves the old video lingering. Only entries the + // bundle format OWNS are candidates: `--out` may point at a directory + // that also holds the user's unrelated files, and those must survive + // the commit (deleting them would be silent data loss). const topLevel = files.filter(f => !f.startsWith('steps/')); const newTopLevelSet = new Set(topLevel); newTopLevelSet.add('meta.json'); // about to land last, do not delete const existing = await readdir(dir).catch(() => [] as string[]); for (const entry of existing) { // Preserve the writer's own scratch dir + the .partial marker - // (we'll re-evaluate .partial at the end of commit). Anything else - // not-listed in the new bundle is stale. + // (we'll re-evaluate .partial at the end of commit). Any other + // bundle-owned entry not-listed in the new bundle is stale. if (entry === '.tmp' || entry === '.partial') continue; if (newTopLevelSet.has(entry)) continue; if (entry === 'steps') continue; // handled below + if (!isBundleOwnedEntry(entry)) continue; // foreign file — never touch await rm(join(dir, entry), { recursive: true, force: true }); } From 93c621c6ba021dc513c31506c35fd632c394b536 Mon Sep 17 00:00:00 2001 From: kshitij-heizen Date: Mon, 6 Jul 2026 01:04:42 +0530 Subject: [PATCH 042/117] fix(rerun): reject explicit IDs with --all and --status/--skip-terminal without --all (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two silent-footgun gaps in test rerun's flag validation: 1. Explicit test IDs combined with --all silently discarded the listed IDs — the --all branch resolves the full project test set and overwrites them — so 'rerun test_abc --all' dispatched a batch rerun of EVERY test in the project, burning rerun/auto-heal credits with no error. Both siblings already guard this exact ambiguity (test run's positional+--all guard, delete-batch's ids+--all guard). 2. --status and --skip-terminal without --all were silently ignored — including INVALID --status values, which were never validated — while the same misuse of rerun's own --filter (and delete-batch's --status) exits 5. All three narrowing filters now share the same guard. Both reject with VALIDATION_ERROR (exit 5) before any network dispatch. Fixes #160 Co-authored-by: Kshitij Bhardwaj Co-authored-by: Claude Fable 5 --- src/commands/test.rerun.spec.ts | 120 ++++++++++++++++++++++++++++++++ src/commands/test.ts | 30 ++++++++ 2 files changed, 150 insertions(+) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index d36a233..129d859 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -280,6 +280,126 @@ describe('runTestRerun — validation', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); }); + it('exit 5 (VALIDATION_ERROR) when explicit test IDs are combined with --all', async () => { + // The --all branch resolves the FULL project test set and overwrites the + // listed ids, so 'rerun test_abc --all' would silently rerun the ENTIRE + // project instead of test_abc — burning rerun/auto-heal credits. The + // guard throws BEFORE any network/dispatch. (Mirrors `test run`'s + // positional+--all guard and delete-batch's ids+--all guard.) + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_abc'], + all: true, + projectId: 'proj_1', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'test-ids' }), + }); + }); + + it('exit 5 (VALIDATION_ERROR) when --status is passed WITHOUT --all', async () => { + // --status is an --all-only narrowing filter. With explicit ids it was + // silently ignored (both tests dispatched, filter dropped) — same + // failure mode as the --filter guard above. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a', 'test_b'], + all: false, + statusFilter: 'failed', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'status' }), + }); + }); + + it('exit 5 (VALIDATION_ERROR) for an INVALID --status value without --all (was silently accepted)', async () => { + // Before the guard, an invalid --status token without --all was never + // even validated: 'rerun test_a --status notastatus' exited 0 while the + // same flag on delete-batch exits 5. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + statusFilter: 'notastatus', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('exit 5 (VALIDATION_ERROR) when --skip-terminal is passed WITHOUT --all', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + skipTerminal: true, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'skip-terminal' }), + }); + }); + it('exit 5 when --all without --project', async () => { const creds = makeCreds(); try { diff --git a/src/commands/test.ts b/src/commands/test.ts index 7fe985d..22d0170 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5980,6 +5980,18 @@ export async function runTestRerun( 'provide at least one , or use --all to rerun all tests in the project', ); } + // Explicit ids + --all is ambiguous: the --all branch resolves the FULL + // project test set and overwrites the listed ids, so the user's narrowing + // intent would be silently replaced by a whole-project batch rerun — + // burning rerun/auto-heal credits. Reject early. (Mirrors `test run`'s + // positional+--all guard and delete-batch's ids+--all data-loss guard.) + if (opts.all && opts.testIds.length > 0) { + throw localValidationError( + 'test-ids', + 'pass either explicit test IDs or --all, not both — --all reruns every test in the ' + + 'project and would ignore the listed IDs. Drop the IDs, or drop --all.', + ); + } if (opts.all && !opts.projectId) { throw localValidationError( 'project', @@ -5998,6 +6010,24 @@ export async function runTestRerun( 'Remove --filter, or add --all --project .', ); } + // --status and --skip-terminal are --all-only narrowing filters with the + // same silent-ignore failure mode as --filter above: without --all the + // explicit ids get reran unfiltered (and an invalid --status value is + // never even validated). Reject both, mirroring the --filter guard. + if (opts.statusFilter !== undefined && !opts.all) { + throw localValidationError( + 'status', + '--status only applies with --all (it narrows which project tests get reran). ' + + 'Remove --status, or add --all --project .', + ); + } + if (opts.skipTerminal && !opts.all) { + throw localValidationError( + 'skip-terminal', + '--skip-terminal only applies with --all (it narrows which project tests get reran). ' + + 'Remove --skip-terminal, or add --all --project .', + ); + } if ( !Number.isInteger(opts.maxConcurrency) || opts.maxConcurrency < 1 || From 759e85c2e046af365c7904376ab577d9e38e0cb3 Mon Sep 17 00:00:00 2001 From: mo01115285816-cyber Date: Sun, 5 Jul 2026 22:34:58 +0300 Subject: [PATCH 043/117] docs(usage): add --debug example + exit codes to help text (#171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'testsprite usage' command help text listed three examples but omitted the --debug flag (a global option useful for diagnosing auth and network issues). It also didn't document the exit codes, which matters for CI/CD scripts that gate on the return value. This PR adds: - A --debug example showing what it traces (HTTP method/path, request id, latency) — useful when debugging 'auth error' or 'transport failure' messages. - An explicit exit-codes section (0 success, 3 auth error, 10 network failure) so users scripting against the CLI know what to expect. Pure documentation improvement — no behavioral change, no new deps. Tested: existing unit tests pass (npm test). Co-authored-by: MOAAMN SAYED --- src/commands/usage.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/commands/usage.ts b/src/commands/usage.ts index d479cab..a97dbba 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -200,7 +200,12 @@ export function createUsageCommand(deps: UsageDeps = {}): Command { '\nExamples:\n' + ' testsprite usage # show balance + plan\n' + ' testsprite usage --output json # machine-readable balance\n' + + ' testsprite usage --debug # trace HTTP method/path, request id, latency\n' + ' testsprite credits # alias for usage\n' + + '\nExit codes:\n' + + ' 0 success (or --dry-run)\n' + + ' 3 auth error — run `testsprite setup` to configure credentials\n' + + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + '\nNote: credit balance requires a backend update to /me. Until shipped,\n' + " check your portal's Billing page (/dashboard/settings/billing) for your balance.", ) From 5ff639978187dc521aecf344de132dfbc5db0ed1 Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:38:16 +0100 Subject: [PATCH 044/117] fix(cli): route interactive prompts and prelude to stderr (keep stdout pure) (#31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interactive prompts (`prompt.ts` — the API-key prompt during `setup`/`auth configure`, the target prompt during `agent install`) wrote the question and masking to STDOUT, and the "Configuring profile …" prelude defaulted to stdout too. On the interactive path that mixes UI text into stdout — and under `--output json` it breaks the contract that stdout is a single JSON document, so a consumer doing `JSON.parse(stdout)` fails. Default both to stderr: prompts and informational preludes are interactive UI, not result data. stdout now carries only the command's result (the §8.1 stdout-purity principle the repo already enforces elsewhere). stderr is still the user's TTY, so prompts remain visible; the secret is still never echoed. Callers that inject explicit streams are unaffected. Adds regression tests: promptText writes the question to stderr by default, and the configure prelude lands on stderr (not the result stdout). --- src/commands/auth.test.ts | 29 ++++++++++++++++++++ src/commands/auth.ts | 5 +++- src/lib/prompt.test.ts | 56 +++++++++++++++++++++++++++++++++++++++ src/lib/prompt.ts | 10 +++++-- 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 8a7ae8b..1605f58 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -278,6 +278,35 @@ describe('runConfigure', () => { expect(capture.prelude.join('')).toContain('Configuring profile "default"'); }); + it('routes the interactive prelude to stderr by default, keeping stdout for the result', async () => { + // Regression: the prelude used to default to process.stdout, polluting the + // result stream (and the JSON document under --output json). With no + // injected preludeWrite/stderr, the default must land on stderr, not stdout. + const stdout: string[] = []; + const errChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + try { + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false }, + { + stdout: line => stdout.push(line), + prompt: { secret: vi.fn(async () => 'sk-typed') }, + fetchImpl: meOkFetch, + credentialsPath, + env: {}, + }, + ); + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + } + expect(errChunks.join('')).toContain('Configuring profile "default"'); + expect(stdout.join('\n')).not.toContain('Configuring profile'); + }); + it('interactive path resolves the endpoint from TESTSPRITE_API_URL without prompting', async () => { const { capture, deps } = makeCapture(); const prompt = { secret: vi.fn(async () => 'sk-typed') }; diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 1ec3e40..bf2eb88 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -75,7 +75,10 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): const env = deps.env ?? process.env; const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const out = makeOutput(opts.output, deps); - const prelude = deps.preludeWrite ?? ((chunk: string) => process.stdout.write(chunk)); + // The "Configuring profile …" prelude is informational, not result data, so + // it defaults to stderr — stdout stays a pure result stream (the configured + // JSON/text), which matters under `--output json` (§8.1 stdout purity). + const prelude = deps.preludeWrite ?? ((chunk: string) => process.stderr.write(chunk)); const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); // Normalize the env endpoint: an empty / whitespace-only TESTSPRITE_API_URL is diff --git a/src/lib/prompt.test.ts b/src/lib/prompt.test.ts index 45275e3..bb8cfb5 100644 --- a/src/lib/prompt.test.ts +++ b/src/lib/prompt.test.ts @@ -69,6 +69,33 @@ describe('promptText', () => { const output = new CaptureStream(); expect(await promptText('? ', { input, output })).toBe('eof-no-newline'); }); + + it('writes the question to stderr by default — keeps stdout pure for --output json', async () => { + // Regression: prompts used to default to stdout, which polluted the JSON + // result on the interactive setup/configure path. Interactive UI belongs + // on stderr. Manually swap the global stream writers (prompt reads + // process.stderr/stdout at call time, so this is reliably intercepted). + const errChunks: string[] = []; + const outChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + (process.stdout as unknown as { write: (c: string) => boolean }).write = c => { + outChunks.push(String(c)); + return true; + }; + try { + await promptText('Q: ', { input: Readable.from(['x\n']) }); // no output → default + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + (process.stdout as unknown as { write: typeof origOut }).write = origOut; + } + expect(errChunks.join('')).toContain('Q: '); + expect(outChunks.join('')).not.toContain('Q: '); + }); }); describe('promptSecret (non-TTY behavior)', () => { @@ -88,6 +115,35 @@ describe('promptSecret (non-TTY behavior)', () => { expect(written).not.toContain('sk-hidden-12345'); }); + it('writes the prompt to stderr by default — keeps stdout pure for --output json', async () => { + // Same regression as promptText: the secret prompt is interactive UI and + // must default to stderr so stdout carries only the command result. Manual + // stream swap (vi.spyOn does not intercept process.stderr.write here). + const errChunks: string[] = []; + const outChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + (process.stdout as unknown as { write: (c: string) => boolean }).write = c => { + outChunks.push(String(c)); + return true; + }; + try { + await promptSecret('Key: ', { input: Readable.from(['sk-x\n']) }); // no output → default + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + (process.stdout as unknown as { write: typeof origOut }).write = origOut; + } + expect(errChunks.join('')).toContain('Key: '); + expect(outChunks.join('')).not.toContain('Key: '); + // The typed secret is never echoed to either real stream. + expect(errChunks.join('')).not.toContain('sk-x'); + expect(outChunks.join('')).not.toContain('sk-x'); + }); + it('honors DEL/backspace before submission', async () => { const DEL = String.fromCharCode(0x7f); const input = Readable.from([`abc${DEL}d\n`]); diff --git a/src/lib/prompt.ts b/src/lib/prompt.ts index 3561b0a..bccead9 100644 --- a/src/lib/prompt.ts +++ b/src/lib/prompt.ts @@ -15,14 +15,20 @@ const pendingPromptInput = new WeakMap(); export async function promptText(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; - const output = streams.output ?? process.stdout; + // Prompts are interactive UI, not data — write the question (and any echo) + // to stderr so stdout carries only the command's result. This keeps + // `--output json` stdout a single pure JSON document even on the interactive + // setup / configure path (§8.1 stdout purity). stderr is still the user's + // TTY, so the prompt remains visible. + const output = streams.output ?? process.stderr; output.write(question); return readLine(input, output, false); } export async function promptSecret(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; - const output = streams.output ?? process.stdout; + // See promptText: interactive prompt + masking go to stderr, not stdout. + const output = streams.output ?? process.stderr; output.write(question); const inputAsTTY = input as Readable & RawModeCapable; From 17650be02c06d9fef2742039a16bb678595bdf89 Mon Sep 17 00:00:00 2001 From: merlinsantiago982-cmd Date: Mon, 6 Jul 2026 03:38:34 +0800 Subject: [PATCH 045/117] fix: harden failure bundle artifact downloads (#60) * block artifact download redirects * redact artifact download urls --------- Co-authored-by: merlinsantiago982-cmd --- src/lib/bundle.test.ts | 56 ++++++++++++++++++++++++++++++++++-------- src/lib/bundle.ts | 20 +++++++++++---- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index 0c0680c..aec451a 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -641,17 +641,24 @@ describe('streamUrlToFile retry', () => { calls++; throw new Error('ENETUNREACH dns lookup failed'); }; - await expect( - streamUrlToFile( - 'https://example.com/x', + let caught: unknown; + + try { + await streamUrlToFile( + 'https://example.com/x?X-Amz-Signature=secret-token', '/tmp/will-not-be-written', fetchImpl as typeof globalThis.fetch, { sleep: noSleep }, - ), - ).rejects.toMatchObject({ + ); + } catch (err) { + caught = err; + } + + expect(caught).toMatchObject({ name: 'TransportError', message: expect.stringContaining('ENETUNREACH'), }); + expect(caught).not.toMatchObject({ message: expect.stringContaining('secret-token') }); expect(calls).toBe(STREAM_URL_MAX_RETRIES); }); @@ -661,17 +668,46 @@ describe('streamUrlToFile retry', () => { calls++; return new Response('Forbidden', { status: 403 }); }; - await expect( - streamUrlToFile( - 'https://example.com/x', + let caught: unknown; + const presignedUrl = 'https://example.com/x?X-Amz-Signature=secret-token#download'; + + try { + await streamUrlToFile( + presignedUrl, '/tmp/will-not-be-written', fetchImpl as typeof globalThis.fetch, { sleep: noSleep }, - ), - ).rejects.toMatchObject({ code: 'UNAVAILABLE' }); + ); + } catch (err) { + caught = err; + } + + expect(caught).toMatchObject({ + code: 'UNAVAILABLE', + details: { status: 403, artifactUrl: 'https://example.com/x' }, + }); + const details = (caught as { details?: Record }).details; + expect(details).not.toHaveProperty('url'); + expect(JSON.stringify(details)).not.toContain('secret-token'); expect(calls).toBe(1); }); + it('disables automatic redirects so unsafe redirect targets cannot bypass URL validation', async () => { + const dir = mkdtempSync(join(tmpdir(), 'stream-test-')); + const dest = join(dir, 'out.bin'); + const redirects: Array = []; + const fetchImpl = async (_url: Parameters[0], init?: RequestInit) => { + redirects.push(init?.redirect); + return new Response('hello', { status: 200 }); + }; + + await streamUrlToFile('https://example.com/x', dest, fetchImpl as typeof globalThis.fetch, { + sleep: noSleep, + }); + + expect(redirects).toEqual(['error']); + }); + it('sleeps between retries', async () => { const sleepDelays: number[] = []; const fetchImpl = async () => { diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index 9c03b07..a3c0808 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -772,17 +772,18 @@ export async function streamUrlToFile( deps?: { sleep?: (ms: number) => Promise }, ): Promise { const sleepFn = deps?.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + const artifactUrl = redactArtifactUrlForDetails(url); for (let attempt = 1; attempt <= STREAM_URL_MAX_RETRIES; attempt++) { let response: Response; try { - response = await fetchImpl(url); + response = await fetchImpl(url, { redirect: 'error' }); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (attempt < STREAM_URL_MAX_RETRIES) { await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + throw new TransportError(`Failed to download presigned URL ${artifactUrl}: ${message}`); } if (!response.ok) { // Non-2xx: the URL itself is bad (expired, unauthorized, not found). @@ -794,7 +795,7 @@ export async function streamUrlToFile( nextAction: 'Re-run `testsprite test failure get`. Presigned URLs in the bundle expire after 15 minutes.', requestId: 'local', - details: { status: response.status, url }, + details: { status: response.status, artifactUrl }, }, }); } @@ -814,7 +815,7 @@ export async function streamUrlToFile( await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + throw new TransportError(`Failed to download presigned URL ${artifactUrl}: ${message}`); } } await mkdir(dirname(filePath), { recursive: true }); @@ -836,11 +837,20 @@ export async function streamUrlToFile( await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed mid-download of ${url}: ${message}`); + throw new TransportError(`Failed mid-download of ${artifactUrl}: ${message}`); } } } +function redactArtifactUrlForDetails(url: string): string { + try { + const parsed = new URL(url); + return `${parsed.origin}${parsed.pathname}`; + } catch { + return ''; + } +} + function isPresignedUrl(value: string): boolean { return value.startsWith('https://'); } From d72290d6d92d2806b33608d67af304782597f753 Mon Sep 17 00:00:00 2001 From: Lexiie Date: Mon, 6 Jul 2026 02:41:12 +0700 Subject: [PATCH 046/117] fix(test): guard default artifact run id path (#172) * fix(test): validate artifact run id default path * fix(test): reject windows dot artifact run ids * fix(test): reject windows dot-suffix artifact run ids * style(test): format artifact run id guard --------- Co-authored-by: Lexiie <28455136+Lexiie@users.noreply.github.com> --- src/commands/test.artifact.spec.ts | 48 ++++++++++++++++++++++++++++++ src/commands/test.ts | 26 +++++++++++++--- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/commands/test.artifact.spec.ts b/src/commands/test.artifact.spec.ts index c058079..c0b9283 100644 --- a/src/commands/test.artifact.spec.ts +++ b/src/commands/test.artifact.spec.ts @@ -21,6 +21,7 @@ import { assertOutDirParentExists, createTestArtifactCommand, createTestCommand, + resolveDefaultArtifactDir, runArtifactGet, runFailureGet, } from './test.js'; @@ -322,6 +323,53 @@ describe('runArtifactGet', () => { } }); + it('rejects path-like runId before auth or fetch when default --out is used', async () => { + const fetchImpl = vi.fn(); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: '../../outside', + failedOnly: false, + }, + { fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it.each([ + '.', + '..', + '. ', + '.. ', + '...', + '.. .', + '. .', + '../outside', + '..\\outside', + 'nested/run', + 'nested\\run', + 'bad\0id', + ])('rejects unsafe default artifact runId segment %j', runId => { + expect(() => resolveDefaultArtifactDir(runId, '/repo')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'run-id' }), + }), + ); + }); + + it('keeps the documented default directory for path-safe runIds', () => { + expect(resolveDefaultArtifactDir(SAMPLE_RUN_ID, '/repo')).toBe( + join('/repo', '.testsprite', 'runs', SAMPLE_RUN_ID), + ); + }); + // ---- --failed-only passed through to writeBundle ---- it('passes --failed-only through to writeBundle (steps filtered to failed ± 1)', async () => { diff --git a/src/commands/test.ts b/src/commands/test.ts index 22d0170..5582603 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -7169,6 +7169,25 @@ export interface ArtifactGetResult { bundle?: WriteBundleResult; } +export function resolveDefaultArtifactDir(runId: string, cwd: string = process.cwd()): string { + requireNonEmpty('run-id', runId); + const windowsNormalizedSegment = runId.replace(/[ .]+$/u, ''); + if ( + windowsNormalizedSegment === '' || + windowsNormalizedSegment === '.' || + windowsNormalizedSegment === '..' || + runId.includes('/') || + runId.includes('\\') || + runId.includes('\0') + ) { + throw localValidationError( + 'run-id', + 'must be a single path-safe segment for the default output directory; pass --out to choose a custom path', + ); + } + return join(cwd, '.testsprite', 'runs', runId); +} + /** * Validate that the parent directory of `resolvedDir` exists and is a * directory. Surfaces `VALIDATION_ERROR` (exit 5) — matches the convention @@ -7218,14 +7237,11 @@ export async function runArtifactGet( deps: TestDeps = {}, ): Promise { const out = makeOutput(opts.output, deps); - const client = makeClient(opts, deps); const { runId } = opts; // Resolve output dir: explicit --out or the default .testsprite/runs// const resolvedDir = - opts.out !== undefined - ? resolveBundleDir(opts.out) - : join(process.cwd(), '.testsprite', 'runs', runId); + opts.out !== undefined ? resolveBundleDir(opts.out) : resolveDefaultArtifactDir(runId); // --dry-run: no network, no disk write. // The client (makeClient) is already wired with createDryRunFetch() when @@ -7271,6 +7287,8 @@ export async function runArtifactGet( await assertOutDirParentExists(resolvedDir); } + const client = makeClient(opts, deps); + // Fetch the run-scoped failure bundle. const { body: context, requestId: fetchRequestId } = await client.getWithMeta( `/runs/${encodeURIComponent(runId)}/failure`, From 2ddb03d5674ed4bfd7d0e7655170ce8b8d2bdd54 Mon Sep 17 00:00:00 2001 From: Resque Date: Sun, 5 Jul 2026 23:41:57 +0400 Subject: [PATCH 047/117] feat(cli): add runtime Node.js version check with clear error message (#11) * feat(cli): add runtime Node.js version check with clear error message * refactor(version-guard): extract to a documented module tested against the real implementation --- src/index.ts | 11 +++++++++ src/version-guard.test.ts | 51 +++++++++++++++++++++++++++++++++++++++ src/version-guard.ts | 41 +++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 src/version-guard.test.ts create mode 100644 src/version-guard.ts diff --git a/src/index.ts b/src/index.ts index 7842ec9..bc15ec5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node + import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; @@ -15,6 +16,16 @@ import { Output, isOutputMode } from './lib/output.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; import { VERSION } from './version.js'; +import { shouldRejectNodeVersion } from './version-guard.js'; + +// Guard: exit early with a clear message on unsupported Node.js versions, +// rather than failing later with a cryptic ESM/runtime error. +if (shouldRejectNodeVersion(process.versions.node)) { + process.stderr.write( + `Error: testsprite requires Node.js >= 20 (found ${process.versions.node}).\nInstall the latest LTS from https://nodejs.org\n`, + ); + process.exit(1); +} const program = new Command(); diff --git a/src/version-guard.test.ts b/src/version-guard.test.ts new file mode 100644 index 0000000..19f0eab --- /dev/null +++ b/src/version-guard.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { + MIN_SUPPORTED_NODE_MAJOR, + parseMajorVersion, + shouldRejectNodeVersion, +} from './version-guard.js'; + +// These tests exercise the REAL guard functions used by src/index.ts, +// imported here rather than re-declared, so a regression in the source is +// actually caught. + +describe('parseMajorVersion', () => { + it('extracts the leading major from a semver string', () => { + expect(parseMajorVersion('20.11.1')).toBe(20); + expect(parseMajorVersion('18.0.0')).toBe(18); + expect(parseMajorVersion('22.3.0')).toBe(22); + }); + + it('returns NaN for a non-numeric version string', () => { + expect(Number.isNaN(parseMajorVersion('not-a-version'))).toBe(true); + }); +}); + +describe('shouldRejectNodeVersion', () => { + it('rejects majors below the supported floor', () => { + expect(shouldRejectNodeVersion('18.19.1')).toBe(true); + expect(shouldRejectNodeVersion('16.20.2')).toBe(true); + expect(shouldRejectNodeVersion('14.21.3')).toBe(true); + }); + + it('accepts the supported floor and above', () => { + expect(shouldRejectNodeVersion('20.0.0')).toBe(false); + expect(shouldRejectNodeVersion('20.11.0')).toBe(false); + expect(shouldRejectNodeVersion('21.0.0')).toBe(false); + expect(shouldRejectNodeVersion('22.1.0')).toBe(false); + }); + + it(`treats exactly ${MIN_SUPPORTED_NODE_MAJOR} as supported (boundary)`, () => { + expect(shouldRejectNodeVersion(`${MIN_SUPPORTED_NODE_MAJOR}.0.0`)).toBe(false); + expect(shouldRejectNodeVersion(`${MIN_SUPPORTED_NODE_MAJOR - 1}.9.9`)).toBe(true); + }); + + it('does not reject an unparseable version (guard never blocks on garbage)', () => { + expect(shouldRejectNodeVersion('not-a-version')).toBe(false); + }); + + it('the running Node satisfies the guard (meta-test)', () => { + // The test suite itself runs on a supported Node, so the guard must pass. + expect(shouldRejectNodeVersion(process.versions.node)).toBe(false); + }); +}); diff --git a/src/version-guard.ts b/src/version-guard.ts new file mode 100644 index 0000000..a7fe464 --- /dev/null +++ b/src/version-guard.ts @@ -0,0 +1,41 @@ +/** + * Node.js runtime version guard. + * + * The CLI targets modern Node (see `engines.node` in package.json). Running on + * an older runtime tends to fail later with a cryptic ESM/syntax error, so the + * entrypoint (`src/index.ts`) uses {@link shouldRejectNodeVersion} to exit early + * with a clear, actionable message instead. + * + * The logic lives here (rather than inline) so it can be unit-tested against the + * real implementation the entrypoint uses — not a copy. + */ + +/** Minimum Node.js major version supported by the CLI (matches package.json `engines.node`). */ +export const MIN_SUPPORTED_NODE_MAJOR = 20; + +/** + * Parse the leading major version number from a Node.js version string. + * + * @param nodeVersion - a dot-separated version string such as `process.versions.node` + * (e.g. `"20.11.1"`). A leading `v` is not expected (Node does not include one here). + * @returns the major version as a number, or `NaN` if the string has no numeric leading segment. + */ +export function parseMajorVersion(nodeVersion: string): number { + return Number(nodeVersion.split('.')[0]); +} + +/** + * Decide whether the given Node.js version is too old to run the CLI. + * + * A version is rejected only when its major number is a real value below + * {@link MIN_SUPPORTED_NODE_MAJOR}. An unparseable string yields `NaN`, which is + * treated as "do not reject" so the guard never blocks on a version string it + * cannot understand (the runtime would surface any real incompatibility itself). + * + * @param nodeVersion - a `process.versions.node` style string (e.g. `"18.19.1"`). + * @returns `true` when the runtime is below the supported floor and should be rejected. + */ +export function shouldRejectNodeVersion(nodeVersion: string): boolean { + const major = parseMajorVersion(nodeVersion); + return !Number.isNaN(major) && major < MIN_SUPPORTED_NODE_MAJOR; +} From 8eb4acff70e216ed6b63bfd19fb1fde7107b5c7f Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Sun, 5 Jul 2026 23:27:00 +0100 Subject: [PATCH 048/117] feat(agent): add windsurf as an install target (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windsurf (Cascade) reads workspace rules from `.windsurf/rules/*.md`. Add it as an own-file agent target so `testsprite agent install --target windsurf` (and `setup --agent windsurf`) installs the TestSprite skills into a Windsurf project. Reworked onto the v0.2.0 multi-skill agent-targets API (pathFor / SKILLS / DEFAULT_SKILLS). Rule files use Cascade frontmatter with `trigger: model_decision` — the equivalent of the Cursor `.mdc` `alwaysApply: false` mode (description shown up front; full body pulled in on relevance). Budget handling: a `.windsurf/rules/*.md` file caps at ~12 K characters and Cascade silently truncates beyond it, which would cut the full ~22 KB verify skill in half. The windsurf target therefore renders the COMPACT body per skill (new `compactBody` flag + `compactBodyFor`): a skill that ships a trimmed codex asset (`testsprite-verify` → ~5 KB) uses it, while a skill whose codex contribution is only a one-liner (`testsprite-onboard`, ~6.5 KB full) keeps its full body — both land well under the cap. `agent.ts` and `renderForTarget` select the same body so installed bytes match the render. Everything else derives from the TARGETS map automatically (agent list, the setup --agent choices, skill-nudge install detection). Updated the hardcoded help strings, the --help snapshot, the agent-targets/agent unit tests (incl. Cascade-frontmatter and per-skill budget tests), the e2e matrix guards / content-integrity (gated on compactBody), and the README/DOCUMENTATION target lists (incl. the --force own-file list). --- DOCUMENTATION.md | 7 ++- README.md | 46 +++++++------- src/commands/agent.test.ts | 23 +++---- src/commands/agent.ts | 22 ++++++- src/lib/agent-targets.test.ts | 54 +++++++++++++++- src/lib/agent-targets.ts | 63 ++++++++++++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 12 ++-- test/e2e/agent-install.e2e.test.ts | 20 +++++- test/e2e/setup.e2e.test.ts | 1 + 9 files changed, 193 insertions(+), 55 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 7a89216..8c0cf1a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -112,16 +112,17 @@ testsprite agent install claude # install the skill for Claude Code testsprite agent install codex # install into AGENTS.md for Codex (managed-section) testsprite agent install cursor # .cursor/rules/testsprite-verify.mdc testsprite agent install cline # .clinerules/testsprite-verify.md +testsprite agent install windsurf # .windsurf/rules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md -testsprite agent list # list all 6 targets with status + mode + path +testsprite agent list # list all 7 targets with status + mode + path ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental). +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental). The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf) backs up the existing file to `.bak` first. ## Command reference diff --git a/README.md b/README.md index 1e20a1d..79f8aea 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ npm install -g @testsprite/testsprite-cli testsprite setup ``` -`testsprite setup` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `antigravity`, `codex`, etc.) — one command, so your agent is wired to verify its own work. Non-interactive (CI / onboarding scripts): +`testsprite setup` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `windsurf`, `antigravity`, `codex`, etc.) — one command, so your agent is wired to verify its own work. Non-interactive (CI / onboarding scripts): ```bash TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude @@ -89,28 +89,28 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test wait` | Block on a `runId` until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index f8ed59f..7da57cf 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -769,12 +769,13 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 6 targets × 2 default skills = 12 rows - expect(json).toHaveLength(12); + // 7 targets × 2 default skills = 14 rows + expect(json).toHaveLength(14); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); + expect(targets).toContain('windsurf'); expect(targets).toContain('antigravity'); expect(targets).toContain('kiro'); expect(targets).toContain('codex'); @@ -916,11 +917,11 @@ describe('createAgentCommand wiring', () => { }); // --------------------------------------------------------------------------- -// All five own-file targets installed at once +// All own-file targets installed at once // --------------------------------------------------------------------------- -describe('runInstall — all five own-file targets', () => { - it('installs all five own-file targets in one invocation', async () => { +describe('runInstall — all own-file targets', () => { + it('installs every own-file target in one invocation', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -930,7 +931,7 @@ describe('runInstall — all five own-file targets', () => { output: 'text', debug: false, dryRun: false, - target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro'], + target: [...OWN_FILE_TARGETS], skills: ['testsprite-verify'], force: false, }, @@ -948,11 +949,11 @@ describe('runInstall — all five own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all five own-file targets +// Dry-run for all six own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the five own-file targets (default 2 skills = 10 would-write lines)', async () => { + it('writes nothing for any of the six own-file targets (default 2 skills = 12 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -962,7 +963,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -972,9 +973,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 5 targets × 2 default skills = 10 would-write lines + // 6 targets × 2 default skills = 12 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(10); + expect(wouldWriteLines.length).toBe(12); }); }); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 1b4e878..0e80a91 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -15,6 +15,7 @@ import { pathFor, loadSkillBodyFor, bodyHash12, + compactBodyFor, buildCodexAggregate, buildSkillMarker, parseSkillMarker, @@ -443,6 +444,21 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr } return b; }; + // Budget-capped own-file targets (e.g. windsurf) render the compact per-skill + // body so the rule file isn't truncated by the agent. Cached separately; must + // match renderForTarget's default selection so written bytes equal the asserted + // render. + const compactBodyCache = new Map(); + const compactBodyForSkill = (skill: string): string => { + let b = compactBodyCache.get(skill); + if (b === undefined) { + b = compactBodyFor(skill); + compactBodyCache.set(skill, b); + } + return b; + }; + const ownFileBodyFor = (t: AgentTarget, skill: string): string => + TARGETS[t].compactBody ? compactBodyForSkill(skill) : bodyForSkill(skill); let codexSectionCache: string | undefined; const getCodexSection = (): string => { if (codexSectionCache === undefined) { @@ -651,7 +667,7 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (abs !== root && !abs.startsWith(root + path.sep)) { throw new CLIError(`refusing to write outside --dir: ${relPath}`, 5); } - const content = renderForTarget(t, skill, bodyForSkill(skill)).content; + const content = renderForTarget(t, skill, ownFileBodyFor(t, skill)).content; if (opts.dryRun) { // Apply the SAME symlink fail-close guard as the real install path @@ -1045,7 +1061,7 @@ function collect(v: string, prev: string[]): string[] { export function createAgentCommand(deps: AgentDeps = {}): Command { const agent = new Command('agent').description( - 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Codex)', + 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Windsurf, Antigravity, Codex)', ); agent @@ -1055,7 +1071,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, kiro, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, codex (comma-separated or repeated)', collect, [], ) diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 38c74e2..0417166 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -80,18 +80,19 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all six required keys', () => { + it('has all seven required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro']); + expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro', 'windsurf']); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, antigravity, kiro, and codex are experimental', () => { + it('cursor, cline, windsurf, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); + expect(TARGETS.windsurf.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); @@ -110,6 +111,7 @@ describe('TARGETS', () => { expect(TARGETS.cursor.mode).toBe('own-file'); expect(TARGETS.cline.mode).toBe('own-file'); expect(TARGETS.kiro.mode).toBe('own-file'); + expect(TARGETS.windsurf.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -293,6 +295,52 @@ describe('renderForTarget("cline")', () => { }); }); +describe('renderForTarget("windsurf")', () => { + const result = renderForTarget('windsurf', 'testsprite-verify', STUB_BODY); + + it('returns the .windsurf/rules path', () => { + expect(result.path).toBe('.windsurf/rules/testsprite-verify.md'); + }); + + it('uses the Cascade frontmatter (trigger: model_decision + description)', () => { + expect(result.content.startsWith('---\n')).toBe(true); + expect(result.content).toContain('trigger: model_decision'); + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); + + it('does NOT carry the Claude/Cursor frontmatter keys', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); // claude key + expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key + }); +}); + +describe('windsurf renders within the rules-file budget', () => { + // Regression: a `.windsurf/rules/*.md` file caps at ~12 K characters and + // Cascade silently truncates beyond that. The full verify body (~22 KB) would + // be cut in half, so windsurf renders the COMPACT body for verify (its trimmed + // codex asset) and the full body for onboard (which already fits). Uses the + // REAL bodies (no stub) so the size reflects what a user receives. + for (const skill of DEFAULT_SKILLS) { + it(`${skill} fits under 12 000 characters`, () => { + const r = renderForTarget('windsurf', skill); + expect(r.content.length).toBeLessThan(12_000); + }); + } + + it('verify uses the compact body (smaller than the full claude render)', () => { + const windsurf = renderForTarget('windsurf', 'testsprite-verify'); + const claude = renderForTarget('claude', 'testsprite-verify'); + expect(windsurf.content.length).toBeLessThan(claude.content.length); + // The full-body-only intro line is absent from the compact body... + expect(claude.content).toContain('The verification loop that flies'); + expect(windsurf.content).not.toContain('The verification loop that flies'); + // ...but the load-bearing command survives. + expect(windsurf.content).toContain('testsprite test run'); + }); +}); + // --------------------------------------------------------------------------- // Content integrity — load-bearing command strings must survive any body trim // --------------------------------------------------------------------------- diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 0273a04..d9a85d5 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -2,7 +2,14 @@ import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { VERSION } from '../version.js'; -export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex' | 'kiro'; +export type AgentTarget = + | 'claude' + | 'cursor' + | 'cline' + | 'antigravity' + | 'codex' + | 'kiro' + | 'windsurf'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -14,11 +21,19 @@ export interface TargetSpec { */ path: string; /** - * 'own-file': the CLI owns the whole file (claude/cursor/cline/antigravity). + * 'own-file': the CLI owns the whole file (claude/cursor/cline/antigravity/windsurf). * 'managed-section': the CLI writes only a sentinel-delimited section inside * a potentially user-authored file (codex target, AGENTS.md). */ mode: 'own-file' | 'managed-section'; + /** + * When true, render the budget-friendly body (see {@link compactBodyFor}) + * instead of the full own-file skill body. Used for own-file targets whose + * rule files are size-capped — currently `windsurf` (`.windsurf/rules/*.md` + * files cap at ~12 K characters and Cascade silently truncates beyond that, + * which would cut the full ~22 KB verify skill in half). + */ + compactBody?: boolean; /** * Wrap a skill body in this target's frontmatter/header. Takes the skill's * `name`+`description` (own-file targets emit them as frontmatter) and the body. @@ -122,6 +137,18 @@ function wrapMdc(_name: string, description: string, body: string): string { return `---\ndescription: ${description}\nalwaysApply: false\n---\n\n${body}\n`; } +/** + * Windsurf (Cascade) reads workspace rules from `.windsurf/rules/*.md` with YAML + * frontmatter. `trigger: model_decision` is the Cascade equivalent of the Cursor + * `.mdc` `alwaysApply: false` mode: only the `description` is surfaced up front, + * and Cascade pulls in the full rule body when the description shows it is + * relevant — exactly the on-demand activation these skills want. (The other + * triggers are `always_on`, `manual`, and `glob`.) + */ +function wrapWindsurf(_name: string, description: string, body: string): string { + return `---\ntrigger: model_decision\ndescription: ${description}\n---\n\n${body}\n`; +} + // --------------------------------------------------------------------------- // Landing paths // --------------------------------------------------------------------------- @@ -144,6 +171,8 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.clinerules/${skill}.md`; case 'kiro': return `.kiro/skills/${skill}/SKILL.md`; + case 'windsurf': + return `.windsurf/rules/${skill}.md`; case 'codex': return 'AGENTS.md'; } @@ -182,6 +211,15 @@ export const TARGETS: Record = { // claude/antigravity, so it shares the wrapSkill wrapper. wrap: wrapSkill, }, + windsurf: { + status: 'experimental', + path: pathFor('windsurf', SKILL_NAME), + mode: 'own-file', + // Windsurf rules files are budget-capped (~12 K chars per `.windsurf/rules/*.md`), + // so render the compact body per skill (see compactBodyFor). + compactBody: true, + wrap: wrapWindsurf, + }, /** * codex target — managed-section mode. * @@ -318,6 +356,24 @@ export function loadSkillBodyFor(skill: string, read: ReadFn = defaultRead): str return readSkillAsset(spec.bodyFile, read); } +/** + * Budget-friendly body for an own-file target whose rule files are size-capped + * (e.g. windsurf). For a skill that ships a trimmed codex asset (`codex.kind === + * 'full'`, e.g. `testsprite-verify` — full body ~22 KB, codex ~5 KB) we render + * that compact asset so the wrapped file stays under the cap. For skills whose + * codex contribution is only a one-liner (`'line'`/`'none'`, e.g. + * `testsprite-onboard`), the one-liner is useless as a standalone rule and the + * full own-file body (~6.5 KB) already fits the budget — so the full body is + * used. + */ +export function compactBodyFor(skill: string, read: ReadFn = defaultRead): string { + const spec = SKILLS[skill]; + if (!spec) throw new Error(`unknown skill: ${skill}`); + return spec.codex.kind === 'full' + ? readSkillAsset(spec.codex.file, read) + : loadSkillBodyFor(skill, read); +} + /** * Resolve a skill's codex (AGENTS.md) contribution as a Markdown string. * 'full' → read the `*.codex.md` asset; 'line' → the inline one-liner; 'none' → ''. @@ -441,7 +497,8 @@ export function renderForTarget( const resolvedBody = body !== undefined ? body : codexContentFor(skill); return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; } - const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); + const resolvedBody = + body !== undefined ? body : spec.compactBody ? compactBodyFor(skill) : loadSkillBodyFor(skill); return { path, content: renderOwnFileWithMarker(t, skill, buildSkillMarker(skill, resolvedBody), resolvedBody), diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 0c387fa..d89e7dc 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -4,7 +4,7 @@ exports[`--help snapshots > agent 1`] = ` "Usage: testsprite agent [options] [command] Install TestSprite guidance into coding-agent config (Claude Code, Cursor, -Cline, Antigravity, Codex) +Cline, Windsurf, Antigravity, Codex) Options: -h, --help display help for command @@ -29,7 +29,7 @@ into a project for a coding agent Options: --target Agent target(s): claude, cursor, cline, antigravity, kiro, - codex (comma-separated or repeated) (default: []) + windsurf, codex (comma-separated or repeated) (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -115,8 +115,8 @@ Options: --from-env Read TESTSPRITE_API_KEY from the environment instead of prompting (default: false) --agent Coding-agent target to install: claude, antigravity, - cursor, cline, kiro, codex (default: claude) (default: - "claude") + cursor, cline, kiro, windsurf, codex (default: claude) + (default: "claude") --no-agent Skip the agent skill install (configure credentials only) --force Overwrite an existing skill file (a .bak backup is kept) --dir Project root for the skill install (default: current @@ -604,8 +604,8 @@ Commands: project Manage TestSprite projects test Inspect TestSprite tests agent Install TestSprite guidance into coding-agent - config (Claude Code, Cursor, Cline, Antigravity, - Codex) + config (Claude Code, Cursor, Cline, Windsurf, + Antigravity, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) help [command] display help for command diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 0513b8b..094aec2 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -169,11 +169,20 @@ describe('content integrity', () => { content.trimStart().startsWith('#'), `cline: should start with a markdown heading`, ).toBe(true); + } else if (target === 'windsurf') { + // Windsurf Cascade frontmatter: trigger + description (no name/alwaysApply) + expect(content.startsWith('---'), `windsurf: should start with ---`).toBe(true); + expect(content).toContain('trigger: model_decision'); + expect(content).toContain('description:'); } - // (b) branding — the renamed H1 must be present + // (b) branding — the renamed H1 must be present in every body variant expect(content).toContain('TestSprite Verification Loop'); - expect(content).toContain('The verification loop that flies'); + // The full-body intro line lives only in the FULL body; compact-body targets + // (e.g. windsurf, budget-capped) ship the trimmed verify body and omit it. + if (!TARGETS[target].compactBody) { + expect(content).toContain('The verification loop that flies'); + } // (c) Load-bearing command strings expect(content, `${target}: missing 'testsprite test run'`).toContain('testsprite test run'); @@ -198,6 +207,10 @@ describe('content integrity', () => { expect(content).toContain('alwaysApply: false'); } else if (target === 'cline') { expect(content.startsWith('---'), `cline/onboard: must NOT start with ---`).toBe(false); + } else if (target === 'windsurf') { + expect(content.startsWith('---'), `windsurf/onboard: should start with ---`).toBe(true); + expect(content).toContain('trigger: model_decision'); + expect(content).toContain('description:'); } // Load-bearing onboard string: the skill body must reference setup @@ -790,7 +803,7 @@ describe('agent list', () => { }>; expect(Array.isArray(parsed)).toBe(true); - // Expected: 5 targets × 2 skills = 10 rows + // Expected: 7 targets × 2 skills = 14 rows const expectedCount = Object.keys(TARGETS).length * DEFAULT_SKILLS.length; expect(parsed.length).toBe(expectedCount); @@ -825,6 +838,7 @@ describe('matrix coverage guard', () => { 'cursor', 'cline', 'kiro', + 'windsurf', 'codex', ]); }); diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index a30c3fb..3b35751 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -228,6 +228,7 @@ describe('matrix coverage guard', () => { 'cursor', 'cline', 'kiro', + 'windsurf', 'codex', ]); }); From 768da46510d4c4d15ce2a70ff50acc7ef0c4c245 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:27:15 -0400 Subject: [PATCH 049/117] feat(cli): honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY behind corporate and CI proxies (#169) * feat(cli): honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY behind corporate and CI proxies * fix(proxy): degrade to default dispatcher when proxy agent init fails instead of crashing startup * fix(proxy): pin undici to ^7.16.0 for Node 20 compatibility (8.x requires Node >=22.19) --- package-lock.json | 53 +++++++++----------------------------- package.json | 1 + src/index.ts | 5 ++++ src/lib/proxy.test.ts | 59 +++++++++++++++++++++++++++++++++++++++++++ src/lib/proxy.ts | 58 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 41 deletions(-) create mode 100644 src/lib/proxy.test.ts create mode 100644 src/lib/proxy.ts diff --git a/package-lock.json b/package-lock.json index 456c498..b254e0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.1.2", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.1.2", + "version": "0.2.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", + "undici": "^7.16.0", "valibot": "^1.4.1" }, "bin": { @@ -1024,9 +1025,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1041,9 +1039,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1058,9 +1053,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1075,9 +1067,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1092,9 +1081,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1109,9 +1095,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1126,9 +1109,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1143,9 +1123,6 @@ "ppc64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1160,9 +1137,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1177,9 +1151,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1194,9 +1165,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1211,9 +1179,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1228,9 +1193,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3855,6 +3817,15 @@ "typescript": ">=4.8.4 <6.1.0" } }, + "node_modules/undici": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index 63c5aac..341e713 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", + "undici": "^7.16.0", "valibot": "^1.4.1" }, "devDependencies": { diff --git a/src/index.ts b/src/index.ts index bc15ec5..f36e4de 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; import { Output, isOutputMode } from './lib/output.js'; +import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; import { VERSION } from './version.js'; @@ -149,6 +150,10 @@ program.hook('preAction', (_thisCommand, actionCommand) => { }); }); +// Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch +// ignores them by default). No-op when no proxy variable is set. +maybeInstallProxyAgent(); + try { await program.parseAsync(process.argv); } catch (err) { diff --git a/src/lib/proxy.test.ts b/src/lib/proxy.test.ts new file mode 100644 index 0000000..5c686af --- /dev/null +++ b/src/lib/proxy.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from 'vitest'; +import { maybeInstallProxyAgent } from './proxy.js'; + +describe('maybeInstallProxyAgent', () => { + it('installs an agent when HTTPS_PROXY is set', () => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }); + + it.each(['https_proxy', 'HTTP_PROXY', 'http_proxy'] as const)( + 'also honors the %s spelling', + name => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { [name]: 'http://proxy.corp.example.com:8080' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }, + ); + + it('does nothing when no proxy variable is set (default path unchanged)', () => { + const install = vi.fn(); + expect(maybeInstallProxyAgent({ env: {}, install })).toBe(false); + expect(maybeInstallProxyAgent({ env: { HTTPS_PROXY: '' }, install })).toBe(false); + expect(install).not.toHaveBeenCalled(); + }); + + it('falls back (returns false, warns, never throws) when installing the agent fails', () => { + // A malformed/unsupported proxy value makes the agent throw at startup; the + // CLI must degrade to a proxy-less dispatcher, not crash every command. + const errs: string[] = []; + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080' }, + install: () => { + throw new Error('unsupported proxy scheme'); + }, + stderr: line => errs.push(line), + }); + expect(installed).toBe(false); + expect(errs.join('\n')).toContain('ignoring proxy environment'); + }); + + it('still installs when NO_PROXY is set (exemptions are applied per request by undici)', () => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080', NO_PROXY: 'localhost,127.0.0.1' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/proxy.ts b/src/lib/proxy.ts new file mode 100644 index 0000000..b536761 --- /dev/null +++ b/src/lib/proxy.ts @@ -0,0 +1,58 @@ +/** + * Proxy support (issue #119): honor HTTPS_PROXY / HTTP_PROXY / NO_PROXY. + * + * Node's built-in fetch (undici) deliberately ignores the proxy environment + * variables, so behind a corporate or CI proxy every request dies with + * `fetch failed` after a full retry cycle. Installing undici's + * `EnvHttpProxyAgent` as the global dispatcher restores the conventional + * behavior (including NO_PROXY exemptions) for every fetch the CLI makes. + * + * Only active when a proxy env var is actually present, so the default path + * stays byte-identical and pays zero startup cost. Dependency note for + * reviewers: this adds `undici` as an explicit runtime dependency (the same + * engine Node already bundles); the alternative, a hand-rolled CONNECT + * tunnel, would re-implement what undici ships and maintains. + */ +import type { Dispatcher } from 'undici'; +import { EnvHttpProxyAgent, setGlobalDispatcher } from 'undici'; + +export interface ProxyDeps { + env?: NodeJS.ProcessEnv; + /** Dispatcher installer. Defaults to undici's setGlobalDispatcher. */ + install?: (agent: Dispatcher) => void; + /** Warning sink. Defaults to `process.stderr`. */ + stderr?: (line: string) => void; +} + +/** + * Install the env-driven proxy dispatcher when any proxy variable is set + * (both canonical upper-case and conventional lower-case spellings). + * Returns whether an agent was installed (observable for tests/debugging). + */ +export function maybeInstallProxyAgent(deps: ProxyDeps = {}): boolean { + const env = deps.env ?? process.env; + const hasProxy = [env.HTTPS_PROXY, env.https_proxy, env.HTTP_PROXY, env.http_proxy].some( + value => typeof value === 'string' && value.length > 0, + ); + if (!hasProxy) return false; + const install = deps.install ?? setGlobalDispatcher; + // EnvHttpProxyAgent reads HTTPS_PROXY/HTTP_PROXY/NO_PROXY itself, per + // request, so NO_PROXY exemptions apply without extra plumbing here. + // + // A malformed or unsupported proxy value (e.g. `socks5://...`) makes the + // agent throw. Because this runs at startup, an unguarded throw would abort + // every command before the CLI's own error handling — so fall back to the + // default (proxy-less) dispatcher and warn instead of crashing. + try { + install(new EnvHttpProxyAgent()); + return true; + } catch (error) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr( + `warning: ignoring proxy environment (could not initialize proxy agent): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return false; + } +} From c6f946e574dc402abf2be1f5d8f56a3a219393cb Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:27:31 -0400 Subject: [PATCH 050/117] feat(cli): non-blocking "new version available" notice (24h-cached npm check, opt-out, CI-safe) (#181) --- DOCUMENTATION.md | 12 ++ src/index.ts | 13 +- src/lib/update-check.test.ts | 231 ++++++++++++++++++++++++++++ src/lib/update-check.ts | 285 +++++++++++++++++++++++++++++++++++ 4 files changed, 540 insertions(+), 1 deletion(-) create mode 100644 src/lib/update-check.test.ts create mode 100644 src/lib/update-check.ts diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 8c0cf1a..0b0fc27 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -455,8 +455,20 @@ These apply to every command: | `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | | `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | | `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | | `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +### Update notice + +Interactive runs print a one-line "new version available" notice on stderr when +a newer release exists. To learn this, the CLI contacts the public npm registry +(`registry.npmjs.org`) at most once per 24 hours; the request carries the +package name only — never your API key, project data, or command line. The +check is skipped in CI, when stderr is not a TTY, under `--output json` / +`--dry-run`, and entirely when `TESTSPRITE_NO_UPDATE_NOTIFIER` is set. Any +failure is silent: the notice can never break or delay a command. This is the +only outbound call the CLI makes besides your configured API endpoint. + ### Scopes API-key scopes gate the write and run surfaces: diff --git a/src/index.ts b/src/index.ts index f36e4de..fb935c8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,7 @@ import { Output, isOutputMode } from './lib/output.js'; import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; +import { maybeNotifyUpdate } from './lib/update-check.js'; import { VERSION } from './version.js'; import { shouldRejectNodeVersion } from './version-guard.js'; @@ -140,14 +141,24 @@ program.hook('preAction', (_thisCommand, actionCommand) => { profile?: string; dryRun?: boolean; }; + const commandPath = commandPathOf(actionCommand); maybeEmitSkillNudge({ - commandPath: commandPathOf(actionCommand), + commandPath, output: isOutputMode(globals.output) ? globals.output : 'text', dryRun: globals.dryRun ?? false, profile: globals.profile ?? 'default', cwd: process.cwd(), env: process.env, }); + + // Best-effort update notice (see lib/update-check.ts): self-gates on the + // opt-out env, CI, TTY, and a 24h cache; the wiring adds the flag-level + // gates the lib cannot see. Skipped for `completion` (its stdout is eval'd + // by shells), under --output json, and under --dry-run. Deliberately not + // awaited: an advisory must never delay the real command. + if (globals.output !== 'json' && globals.dryRun !== true && commandPath !== 'completion') { + void maybeNotifyUpdate(); + } }); // Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch diff --git a/src/lib/update-check.test.ts b/src/lib/update-check.test.ts new file mode 100644 index 0000000..dbce172 --- /dev/null +++ b/src/lib/update-check.test.ts @@ -0,0 +1,231 @@ +/** + * Unit tests for the update notice (issue #122). Every effect is injected: + * no real network, filesystem, clock, or TTY is touched. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { UpdateCheckDeps } from './update-check.js'; +import { + UPDATE_CHECK_OPT_OUT_ENV, + UPDATE_CHECK_TTL_MS, + compareSemver, + fetchLatestVersion, + maybeNotifyUpdate, + shouldCheckForUpdate, +} from './update-check.js'; + +/** In-memory fs + deterministic clock harness for the cache round-trip. */ +function makeHarness(overrides: UpdateCheckDeps = {}) { + const files = new Map(); + const stderrLines: string[] = []; + const deps: UpdateCheckDeps = { + env: {}, + now: () => 1_000_000, + cachePath: '/fake/.testsprite/update-check.json', + readFile: path => { + const content = files.get(path); + if (content === undefined) throw new Error('ENOENT'); + return content; + }, + writeFile: (path, content) => { + files.set(path, content); + }, + mkdir: () => undefined, + isTTY: true, + stderr: line => stderrLines.push(line), + currentVersion: '0.2.0', + fetchImpl: async () => + new Response(JSON.stringify({ version: '0.2.0' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ...overrides, + }; + return { deps, files, stderrLines }; +} + +describe('shouldCheckForUpdate gates', () => { + it('opt-out env set to any non-empty value disables (even "0")', () => { + const { deps } = makeHarness({ env: { [UPDATE_CHECK_OPT_OUT_ENV]: '0' } }); + expect(shouldCheckForUpdate(deps)).toBe(false); + }); + + it('CI set disables; CI="false" re-enables', () => { + expect(shouldCheckForUpdate(makeHarness({ env: { CI: 'true' } }).deps)).toBe(false); + expect(shouldCheckForUpdate(makeHarness({ env: { CI: '' } }).deps)).toBe(false); + expect(shouldCheckForUpdate(makeHarness({ env: { CI: 'false' } }).deps)).toBe(true); + }); + + it('non-TTY stderr disables', () => { + expect(shouldCheckForUpdate(makeHarness({ isTTY: false }).deps)).toBe(false); + }); + + it('a fresh cache suppresses; a stale cache does not', () => { + const fresh = makeHarness(); + fresh.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 1_000_000 - UPDATE_CHECK_TTL_MS + 5_000 }), + ); + expect(shouldCheckForUpdate(fresh.deps)).toBe(false); + + const stale = makeHarness(); + stale.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 1_000_000 - UPDATE_CHECK_TTL_MS - 5_000 }), + ); + expect(shouldCheckForUpdate(stale.deps)).toBe(true); + }); + + it('missing, corrupt, wrong-shape, or future-stamped caches count as stale', () => { + expect(shouldCheckForUpdate(makeHarness().deps)).toBe(true); // missing + const corrupt = makeHarness(); + corrupt.files.set('/fake/.testsprite/update-check.json', '{not json'); + expect(shouldCheckForUpdate(corrupt.deps)).toBe(true); + const wrongShape = makeHarness(); + wrongShape.files.set('/fake/.testsprite/update-check.json', JSON.stringify({ nope: true })); + expect(shouldCheckForUpdate(wrongShape.deps)).toBe(true); + const future = makeHarness(); + future.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 9_999_999_999 }), + ); + expect(shouldCheckForUpdate(future.deps)).toBe(true); + }); +}); + +describe('fetchLatestVersion', () => { + it('returns the version from a valid registry body', async () => { + const { deps } = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '1.2.3' }), { status: 200 }), + }); + await expect(fetchLatestVersion(deps)).resolves.toBe('1.2.3'); + }); + + it('returns undefined on non-2xx, thrown fetch, and wrong-shape body', async () => { + const notOk = makeHarness({ fetchImpl: async () => new Response('nope', { status: 500 }) }); + await expect(fetchLatestVersion(notOk.deps)).resolves.toBeUndefined(); + + const throwing = makeHarness({ + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + await expect(fetchLatestVersion(throwing.deps)).resolves.toBeUndefined(); + + const wrongShape = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ notVersion: 1 }), { status: 200 }), + }); + await expect(fetchLatestVersion(wrongShape.deps)).resolves.toBeUndefined(); + }); +}); + +describe('compareSemver', () => { + it('orders numerically and treats prerelease as older than its release', () => { + expect(compareSemver('0.2.0', '0.3.0')).toBe(-1); + expect(compareSemver('1.0.0', '0.9.9')).toBe(1); + expect(compareSemver('0.2.0', '0.2.0')).toBe(0); + expect(compareSemver('0.10.0', '0.9.0')).toBe(1); // numeric, not lexicographic + expect(compareSemver('1.0.0-rc.1', '1.0.0')).toBe(-1); + expect(compareSemver('1.0.0', '1.0.0-rc.1')).toBe(1); + }); + + it('unparseable input on either side compares as 0 (never a false notice)', () => { + expect(compareSemver('garbage', '1.0.0')).toBe(0); + expect(compareSemver('1.0.0', '')).toBe(0); + }); +}); + +describe('maybeNotifyUpdate', () => { + it('prints exactly one stderr line naming both versions when newer, and stamps the cache', async () => { + const harness = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '0.3.1' }), { status: 200 }), + }); + await maybeNotifyUpdate(harness.deps); + expect(harness.stderrLines).toHaveLength(1); + expect(harness.stderrLines[0]).toContain('0.2.0 -> 0.3.1'); + expect(harness.stderrLines[0]).toContain(UPDATE_CHECK_OPT_OUT_ENV); + const cache = JSON.parse(harness.files.get('/fake/.testsprite/update-check.json')!) as { + lastCheckMs: number; + latestKnown?: string; + }; + expect(cache.lastCheckMs).toBe(1_000_000); + expect(cache.latestKnown).toBe('0.3.1'); + }); + + it('stays silent on an equal or older registry version', async () => { + const equal = makeHarness(); + await maybeNotifyUpdate(equal.deps); + expect(equal.stderrLines).toHaveLength(0); + + const older = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '0.1.9' }), { status: 200 }), + }); + await maybeNotifyUpdate(older.deps); + expect(older.stderrLines).toHaveLength(0); + }); + + it('a failed probe stays silent but still stamps the cache (retry once per TTL)', async () => { + const harness = makeHarness({ + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + await maybeNotifyUpdate(harness.deps); + expect(harness.stderrLines).toHaveLength(0); + const cache = JSON.parse(harness.files.get('/fake/.testsprite/update-check.json')!) as { + lastCheckMs: number; + latestKnown?: string; + }; + expect(cache.lastCheckMs).toBe(1_000_000); + expect(cache.latestKnown).toBeUndefined(); + }); + + it('does nothing when a gate blocks (no fetch fired)', async () => { + const fetchImpl = vi.fn(async () => new Response('{}', { status: 200 })); + const harness = makeHarness({ env: { CI: '1' }, fetchImpl }); + await maybeNotifyUpdate(harness.deps); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(harness.stderrLines).toHaveLength(0); + }); + + it('never rejects, even when every injected dependency throws', async () => { + const harness = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '9.9.9' }), { status: 200 }), + writeFile: () => { + throw new Error('EROFS'); + }, + stderr: () => { + throw new Error('broken stderr sink'); + }, + }); + await expect(maybeNotifyUpdate(harness.deps)).resolves.toBeUndefined(); + }); + + it('uses the default fs readers/writers when none are injected (real cache round-trip)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'update-check-')); + // Point cachePath at a not-yet-existing subdir so the default mkdir + // (recursive) and writeFile arrows both run, and the first read (file + // absent) exercises the default readFile arrow's ENOENT path. + const cachePath = join(dir, 'nested', 'update-check.json'); + const stderrLines: string[] = []; + try { + await maybeNotifyUpdate({ + env: {}, + now: () => 1_000_000, + isTTY: true, + currentVersion: '0.0.1', + cachePath, + stderr: line => stderrLines.push(line), + fetchImpl: async () => new Response(JSON.stringify({ version: '9.9.9' }), { status: 200 }), + }); + // Cache was persisted by the default writeFile through the default mkdir. + expect(JSON.parse(readFileSync(cachePath, 'utf8'))).toMatchObject({ latestKnown: '9.9.9' }); + expect(stderrLines.join('\n')).toContain('0.0.1 -> 9.9.9'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/update-check.ts b/src/lib/update-check.ts new file mode 100644 index 0000000..e928913 --- /dev/null +++ b/src/lib/update-check.ts @@ -0,0 +1,285 @@ +/** + * Non-blocking "new version available" notice (issue #122), following the + * pattern of the gh and npm CLIs: at most one npm-registry probe per 24 hours, + * result cached on disk, advisory printed to stderr so stdout stays parseable. + * + * Behavior (`maybeNotifyUpdate`): + * 1. Gate through `shouldCheckForUpdate`; every gate below must pass. + * 2. Probe the npm registry for the `latest` dist-tag (1.5s hard timeout). + * 3. Stamp the cache with `lastCheckMs` (plus `latestKnown` when the probe + * succeeded) so the next 24h of invocations skip the network entirely. + * The stamp happens even after a failed probe: a dead registry must not + * trigger a retry on every command. + * 4. When `latest` is strictly newer than the running version, write exactly + * one advisory line to stderr. The function never throws or rejects and + * never alters the exit status of the command it rides along with. + * + * Gates, in order (`shouldCheckForUpdate`): + * - `TESTSPRITE_NO_UPDATE_NOTIFIER` set to any non-empty value: opted out. + * Presence-style, mirroring gh's GH_NO_UPDATE_NOTIFIER: even "0" disables. + * - `CI` set to anything except the literal "false": CI logs are not the + * place for update nags. `CI=false` explicitly re-enables the notice. + * - stderr is not a TTY: piped or redirected output stays clean. + * - the on-disk cache is fresh (last probe within the TTL). A missing, + * unreadable, corrupt, or wrong-shape cache counts as stale, and so does a + * `lastCheckMs` in the future (clock rollback or corrupt data). + * + * Why not the npm `update-notifier` package: this CLI's runtime dependency + * budget is commander + valibot only (package.json). `update-notifier` would + * add a transitive dependency tree for what Node 20 already ships as + * primitives (fetch, AbortSignal.timeout, sync fs). Owning the ~100 lines + * keeps the install size flat and every effect injectable for tests. + * + * All effects (env, network, clock, fs, tty, stderr sink) are injectable via + * `UpdateCheckDeps`, the same dependency-injection style as `skill-nudge.ts`. + */ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import * as v from 'valibot'; +import { VERSION } from '../version.js'; +import type { FetchImpl } from './http.js'; + +/** Re-check interval: 24 hours, expressed in milliseconds. */ +export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000; + +/** + * Env var that disables the update notice entirely. Presence-style: any + * non-empty value (including "0") opts out. Deliberately stricter than the + * truthy-style `TESTSPRITE_NO_SKILL_WARNING` because it matches the + * convention users already know from gh's GH_NO_UPDATE_NOTIFIER. + */ +export const UPDATE_CHECK_OPT_OUT_ENV = 'TESTSPRITE_NO_UPDATE_NOTIFIER'; + +/** + * Hard cap for the registry probe, in milliseconds. The probe rides along a + * real command, so unlike the 120s API budget in `http.ts` it gets a tiny + * window: a slow registry means "no update info", never a visible delay. + */ +const REGISTRY_TIMEOUT_MS = 1_500; + +/** + * npm registry `latest` dist-tag endpoint for this package. Package name from + * package.json (`@testsprite/testsprite-cli`); the scope separator must be + * URL-encoded as %2F per the npm registry API. + */ +const REGISTRY_LATEST_URL = 'https://registry.npmjs.org/@testsprite%2Ftestsprite-cli/latest'; + +/** On-disk cache shape at `cachePath`; unknown keys are stripped on read. */ +const UPDATE_CHECK_CACHE_SCHEMA = v.object({ + lastCheckMs: v.number(), + latestKnown: v.optional(v.string()), +}); + +export type UpdateCheckCache = v.InferOutput; + +/** Minimal slice of the registry response the notice needs. */ +const REGISTRY_LATEST_BODY_SCHEMA = v.object({ version: v.string() }); + +export interface UpdateCheckDeps { + env?: NodeJS.ProcessEnv; + fetchImpl?: FetchImpl; + /** Clock, epoch milliseconds. */ + now?: () => number; + /** Cache file; lives next to credentials/config under ~/.testsprite. */ + cachePath?: string; + readFile?: (path: string) => string; + writeFile?: (path: string, content: string) => void; + /** Must create missing parent directories (recursive). */ + mkdir?: (dir: string) => void; + /** Whether stderr is an interactive terminal. */ + isTTY?: boolean; + /** Sink for the single advisory line. */ + stderr?: (line: string) => void; + /** Version the running binary reports. */ + currentVersion?: string; +} + +type ResolvedUpdateCheckDeps = Required; + +function resolveUpdateCheckDeps(deps: UpdateCheckDeps): ResolvedUpdateCheckDeps { + return { + env: deps.env ?? process.env, + fetchImpl: deps.fetchImpl ?? globalThis.fetch, + now: deps.now ?? Date.now, + cachePath: deps.cachePath ?? join(homedir(), '.testsprite', 'update-check.json'), + readFile: deps.readFile ?? ((path: string) => readFileSync(path, 'utf8')), + writeFile: + deps.writeFile ?? ((path: string, content: string) => writeFileSync(path, content, 'utf8')), + mkdir: + deps.mkdir ?? + ((dir: string) => { + mkdirSync(dir, { recursive: true }); + }), + isTTY: deps.isTTY ?? process.stderr.isTTY === true, + stderr: deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)), + currentVersion: deps.currentVersion ?? VERSION, + }; +} + +/** + * Read and validate the cache file. Every failure mode (missing file, + * unreadable file, invalid JSON, wrong shape) returns undefined, which the + * caller treats as "stale, probe again". + */ +function readUpdateCheckCache(resolved: ResolvedUpdateCheckDeps): UpdateCheckCache | undefined { + try { + const raw = resolved.readFile(resolved.cachePath); + const body: unknown = JSON.parse(raw); + const parsed = v.safeParse(UPDATE_CHECK_CACHE_SCHEMA, body); + return parsed.success ? parsed.output : undefined; + } catch { + // Missing or unreadable cache: treat as stale. + return undefined; + } +} + +/** + * Persist the cache, creating the parent directory when missing. Best-effort: + * every error (read-only home, quota, fs races) is swallowed. A failed write + * only means the next invocation probes the registry again. + */ +function writeUpdateCheckCache(resolved: ResolvedUpdateCheckDeps, cache: UpdateCheckCache): void { + try { + resolved.mkdir(dirname(resolved.cachePath)); + resolved.writeFile(resolved.cachePath, `${JSON.stringify(cache)}\n`); + } catch { + // Cache persistence is optional; never surface fs errors to the command. + } +} + +/** + * True when every gate documented in the module header passes: no opt-out + * env, not CI (unless CI=false), stderr is a TTY, and the cached check is + * stale or absent. Order matters: the cheap env gates run before any fs read. + */ +export function shouldCheckForUpdate(deps: UpdateCheckDeps = {}): boolean { + const resolved = resolveUpdateCheckDeps(deps); + + const optOutValue = resolved.env[UPDATE_CHECK_OPT_OUT_ENV]; + if (optOutValue !== undefined && optOutValue !== '') return false; + + // Any set CI value except the literal "false" counts as CI. The empty + // string still signals a CI-managed environment; silence wins in doubt. + const ciValue = resolved.env.CI; + if (ciValue !== undefined && ciValue !== 'false') return false; + + if (!resolved.isTTY) return false; + + const cache = readUpdateCheckCache(resolved); + if (cache !== undefined) { + const elapsedMs = resolved.now() - cache.lastCheckMs; + // A negative elapsed (lastCheckMs in the future) means clock rollback or + // corrupt data: treat as stale instead of suppressing the check forever. + // A NaN lastCheckMs fails both comparisons and lands on stale too. + if (elapsedMs >= 0 && elapsedMs < UPDATE_CHECK_TTL_MS) return false; + } + + return true; +} + +/** + * Probe the npm registry for the `latest` dist-tag version of this package. + * Hard 1.5s timeout via AbortSignal.timeout. ANY failure (network error, + * timeout, non-2xx status, invalid JSON, wrong shape) resolves to undefined; + * this function never rejects. + */ +export async function fetchLatestVersion(deps: UpdateCheckDeps = {}): Promise { + const resolved = resolveUpdateCheckDeps(deps); + try { + const response = await resolved.fetchImpl(REGISTRY_LATEST_URL, { + signal: AbortSignal.timeout(REGISTRY_TIMEOUT_MS), + }); + if (!response.ok) return undefined; + const body: unknown = await response.json(); + const parsed = v.safeParse(REGISTRY_LATEST_BODY_SCHEMA, body); + return parsed.success ? parsed.output.version : undefined; + } catch { + // Offline, DNS failure, abort, or a non-JSON body: no update info. + return undefined; + } +} + +interface ParsedSemver { + major: number; + minor: number; + patch: number; + hasPrerelease: boolean; +} + +/** + * x.y.z with optional prerelease (after "-") and optional build metadata + * (after "+"), per the semver 2.0.0 grammar. A leading "v" is tolerated + * because humans type it; the npm registry never returns one. + */ +const SEMVER_RE = /^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/; + +function parseSemver(version: string): ParsedSemver | undefined { + const match = SEMVER_RE.exec(version.trim()); + if (!match) return undefined; + const [, majorRaw, minorRaw, patchRaw, prerelease] = match; + if (majorRaw === undefined || minorRaw === undefined || patchRaw === undefined) return undefined; + return { + major: Number(majorRaw), + minor: Number(minorRaw), + patch: Number(patchRaw), + hasPrerelease: prerelease !== undefined, + }; +} + +/** + * Compare two semver strings numerically. Returns -1 when versionA is older + * than versionB, 1 when newer, 0 when equal. A version carrying a prerelease + * tag sorts OLDER than the plain release with the same x.y.z core. Prerelease + * identifiers themselves are not ranked (two prereleases on the same core + * compare as 0): the registry `latest` tag points at releases, so identifier + * ordering never decides whether the notice fires. Unparseable input on + * either side compares as 0, so garbage can never produce a false notice. + */ +export function compareSemver(versionA: string, versionB: string): number { + const left = parseSemver(versionA); + const right = parseSemver(versionB); + if (left === undefined || right === undefined) return 0; + if (left.major !== right.major) return left.major < right.major ? -1 : 1; + if (left.minor !== right.minor) return left.minor < right.minor ? -1 : 1; + if (left.patch !== right.patch) return left.patch < right.patch ? -1 : 1; + if (left.hasPrerelease !== right.hasPrerelease) return left.hasPrerelease ? -1 : 1; + return 0; +} + +/** + * Fire-and-forget update notice. Gates, probes, stamps the cache, and prints + * at most one stderr line when the registry version is strictly newer than + * the running one. Never throws and never rejects: any failure in any + * injected dependency (clock, fs, network, the stderr sink itself) is + * swallowed, because an advisory must never break or delay a real command. + */ +export async function maybeNotifyUpdate(deps: UpdateCheckDeps = {}): Promise { + try { + const resolved = resolveUpdateCheckDeps(deps); + if (!shouldCheckForUpdate(resolved)) return; + + const latest = await fetchLatestVersion(resolved); + + // Stamp even on a failed probe so a dead registry is retried at most + // once per TTL window, not on every invocation. + writeUpdateCheckCache(resolved, { + lastCheckMs: resolved.now(), + ...(latest === undefined ? {} : { latestKnown: latest }), + }); + + if (latest === undefined) return; + if (compareSemver(latest, resolved.currentVersion) !== 1) return; + + // User-facing advisory copy (exact format specified by issue #122), not a + // diagnostic log line; stderr keeps stdout parseable for scripts. + resolved.stderr( + `A new version of testsprite-cli is available: ${resolved.currentVersion} -> ${latest}. ` + + `Run npm install -g @testsprite/testsprite-cli to update. ` + + `(Disable with ${UPDATE_CHECK_OPT_OUT_ENV}=1)`, + ); + } catch { + // An update notice must never break, delay, or alter the exit status of + // the command it accompanies. Swallow everything. + } +} From 4b724619e27a360a00e6c578db3e5068bb566d91 Mon Sep 17 00:00:00 2001 From: Resque Date: Mon, 6 Jul 2026 02:27:46 +0400 Subject: [PATCH 051/117] feat(cli): add 'test flaky' repeat-run flaky-test detector (#132) * feat(cli): add 'test flaky' repeat-run flaky-test detector * fix(flaky): cap --runs at 10 per maintainer scope (#115) Rescope the flaky detector's --runs bound from 1-100 to 1-10 as requested in the #115 triage: uncapped FE replays amplify free executions. Updates the MAX_FLAKY_RUNS constant (which drives the validation, error message, and --runs help text), docs, changelog, and the runs-bound tests. Regenerates the help snapshot, which also adds the previously-missing 'test flaky' entry. * test(snapshot): refresh flaky help snapshot after rebase onto main Rebasing onto current main (which added the global --request-timeout option, #17) changes the 'Global options' line rendered in the test flaky --help output. Regenerate the snapshot so the help snapshot test stays green on CI. * docs(changelog): resolve leftover merge-conflict markers (keep JUnit + flaky 1-10) --- CHANGELOG.md | 1 + DOCUMENTATION.md | 23 ++ src/commands/test.flaky.spec.ts | 389 ++++++++++++++++++ src/commands/test.test.ts | 1 + src/commands/test.ts | 234 +++++++++++ src/lib/flaky.test.ts | 105 +++++ src/lib/flaky.ts | 133 ++++++ test/__snapshots__/help.snapshot.test.ts.snap | 41 ++ test/help.snapshot.test.ts | 1 + 9 files changed, 928 insertions(+) create mode 100644 src/commands/test.flaky.spec.ts create mode 100644 src/lib/flaky.test.ts create mode 100644 src/lib/flaky.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a4ab9b7..4f1a0a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ### Added - **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. +- **`testsprite test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs `, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky --runs 5 || exit 1`). Flags: `--runs ` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout ` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits. ## [0.2.0] - 2026-06-29 diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 0b0fc27..fb62cae 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -403,6 +403,29 @@ Flags: A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. +#### `testsprite test flaky ` + +Detect a **flaky** test by replaying it several times and reporting how often it passes. Each attempt is a rerun with auto-heal **off** (a strict verbatim replay), so healed drift can't disguise a nondeterministic pass/fail — this measures the replay stability of the saved script against the configured URL. Frontend replays are free verbatim script replays; backend tests re-run their dependency closure and may cost credits (a one-line stderr advisory is printed before the run). + +```bash +# Replay 10 times and print a stability score +testsprite test flaky test_xxxxxxxx --runs 10 + +# Fast "is it flaky at all?" — stop at the first non-passing attempt +testsprite test flaky test_xxxxxxxx --runs 10 --until-fail + +# Machine-readable stability report for CI +testsprite test flaky test_xxxxxxxx --runs 10 --output json +``` + +Flags: + +- `--runs ` — number of replays (1–10, default 5). +- `--until-fail` — stop at the first attempt that does not pass. +- `--timeout ` — per-attempt polling deadline (same semantics as `test wait`). + +`--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. + #### `testsprite test wait ` Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. diff --git a/src/commands/test.flaky.spec.ts b/src/commands/test.flaky.spec.ts new file mode 100644 index 0000000..0c4450e --- /dev/null +++ b/src/commands/test.flaky.spec.ts @@ -0,0 +1,389 @@ +/** + * Unit tests for `test flaky` — the repeat-run flaky-test detector. + * + * All HTTP is mocked via `makeFlakyFetch`. The polling loop's sleep is injected + * through `TestDeps.sleep` to avoid real delays. Each rerun POST returns a + * unique runId; each run GET returns a terminal status scripted per attempt, + * so a test can assert stable / flaky / failing verdicts deterministically. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { CLIError, ApiError } from '../lib/errors.js'; +import type { FlakyReport } from '../lib/flaky.js'; +import type { FetchImpl } from '../lib/http.js'; +import { runFlaky } from './test.js'; + +type FetchInput = Parameters[0]; +type RunStatus = 'passed' | 'failed' | 'blocked' | 'cancelled'; + +function urlOf(input: FetchInput): string { + return typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; +} + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +/** + * Build a fetch that: + * - GET /tests/{id} → the test record ('frontend' | 'backend') + * - POST /tests/{id}/runs/rerun → a queued rerun with runId run_ (n increments) + * - GET /runs/run_ → a terminal run with statuses[k-1] + * + * `notFoundOnTrigger` makes the rerun POST return 404 (no replayable run). + */ +function makeFlakyFetch(opts: { + statuses: RunStatus[]; + testType?: 'frontend' | 'backend'; + notFoundOnTrigger?: boolean; +}): { fetchImpl: FetchImpl; triggerCount: () => number } { + let triggers = 0; + const testType = opts.testType ?? 'frontend'; + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = urlOf(input); + const method = (init.method ?? 'GET').toUpperCase(); + + if (method === 'GET' && /\/tests\/[^/]+$/.test(url.split('?')[0]!)) { + return jsonResponse(200, { + id: 'test_x', + projectId: 'project_abc', + name: 'sample', + type: testType, + createdFrom: 'portal', + status: 'passed', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }); + } + + if (method === 'POST' && url.includes('/runs/rerun')) { + if (opts.notFoundOnTrigger) { + return jsonResponse(404, { + error: { + code: 'NOT_FOUND', + message: 'no replayable run', + nextAction: 'run it', + requestId: 'req_1', + details: {}, + }, + }); + } + triggers += 1; + return jsonResponse(200, { + runId: `run_${triggers}`, + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + }); + } + + const runMatch = /\/runs\/(run_\d+)/.exec(url); + if (method === 'GET' && runMatch) { + const runId = runMatch[1]!; + const idx = Number(runId.replace('run_', '')) - 1; + const status = opts.statuses[idx] ?? 'passed'; + return jsonResponse(200, { + runId, + testId: 'test_x', + projectId: 'project_abc', + userId: 'user_1', + status, + source: 'cli', + createdAt: '2026-06-03T10:00:00.000Z', + startedAt: '2026-06-03T10:00:01.000Z', + finishedAt: '2026-06-03T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'rerun:prior', + failedStepIndex: status === 'passed' ? null : 2, + failureKind: status === 'passed' ? null : 'assertion', + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 5, passedCount: 5, failedCount: 0 }, + }); + } + + return jsonResponse(404, { + error: { code: 'NOT_FOUND', message: 'unmatched', requestId: 'x', details: {} }, + }); + }) as FetchImpl; + + return { fetchImpl, triggerCount: () => triggers }; +} + +function makeCreds(): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-flaky-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync( + credentialsPath, + `[default]\napi_url = http://localhost:13509\napi_key = sk-user-test\n`, + { + mode: 0o600, + }, + ); + return { credentialsPath }; +} + +const instantSleep = (): Promise => Promise.resolve(); + +function makeDeps(fetchImpl: FetchImpl): { + deps: { + credentialsPath: string; + fetchImpl: FetchImpl; + sleep: () => Promise; + stdout: (l: string) => void; + stderr: (l: string) => void; + }; + stdout: string[]; + stderr: string[]; +} { + const stdout: string[] = []; + const stderr: string[] = []; + const { credentialsPath } = makeCreds(); + return { + deps: { + credentialsPath, + fetchImpl, + sleep: instantSleep, + stdout: (l: string) => stdout.push(l), + stderr: (l: string) => stderr.push(l), + }, + stdout, + stderr, + }; +} + +// --------------------------------------------------------------------------- +// Surface +// --------------------------------------------------------------------------- + +describe('createTestCommand — flaky subcommand exposed', () => { + it('exposes flaky with its flags', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const flaky = test.commands.find(c => c.name() === 'flaky'); + expect(flaky).toBeDefined(); + const flagNames = flaky!.options.map(o => o.long); + expect(flagNames).toContain('--runs'); + expect(flagNames).toContain('--until-fail'); + expect(flagNames).toContain('--timeout'); + }); +}); + +// --------------------------------------------------------------------------- +// Behavior +// --------------------------------------------------------------------------- + +describe('runFlaky', () => { + it('reports STABLE and exits 0 when every attempt passes', async () => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: ['passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + const report = (await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + )) as FlakyReport; + expect(report.verdict).toBe('stable'); + expect(report.runs).toBe(3); + expect(triggerCount()).toBe(3); + }); + + it('reports FLAKY and throws exit 1 on a mix of pass/fail', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'failed', 'passed'] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).exitCode).toBe(1); + expect((err as CLIError).message).toContain('flaky'); + }); + + it('reports FAILING and throws exit 1 when no attempt passes', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['failed', 'failed'] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).message).toContain('failing'); + }); + + it('--until-fail stops at the first non-passing attempt', async () => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: ['passed', 'failed', 'passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 5, + untilFail: true, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + // Stopped after attempt 2 (the failure) — only 2 triggers fired. + expect(triggerCount()).toBe(2); + expect(err).toBeInstanceOf(CLIError); + }); + + it('prints a backend credit advisory to stderr', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'passed'], testType: 'backend' }); + const { deps, stderr } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ); + expect(stderr.some(l => l.includes('backend test') && l.includes('credits'))).toBe(true); + }); + + it('emits a machine-readable JSON stability report', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'failed', 'passed'] }); + const { deps, stdout } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch(() => undefined); // swallow the exit-1 throw; we only assert stdout + const parsed = JSON.parse(stdout.join('\n')) as FlakyReport; + expect(parsed.testId).toBe('test_x'); + expect(parsed.runs).toBe(3); + expect(parsed.passed).toBe(2); + expect(parsed.verdict).toBe('flaky'); + expect(parsed.failures).toHaveLength(1); + expect(parsed.failures[0]!.failureKind).toBe('assertion'); + }); + + it('throws exit 4 when the test has no replayable run', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [], notFoundOnTrigger: true }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('NOT_FOUND'); + }); + + it('rejects --runs below the range (0) with a validation error (exit 5)', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 0, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(5); + }); + + it('rejects --runs above the cap (11) with a validation error (exit 5)', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 11, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(5); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 38c16e5..6d8d734 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -124,6 +124,7 @@ describe('createTestCommand — surface', () => { 'delete-batch', 'diff', 'failure', + 'flaky', 'get', 'list', 'plan', diff --git a/src/commands/test.ts b/src/commands/test.ts index 5582603..508d8be 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -75,6 +75,14 @@ import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; import { loadConfig } from '../lib/config.js'; +import { + flakyExitCode, + renderFlakyText, + summarizeFlaky, + type FlakyAttempt, + type FlakyOutcome, + type FlakyReport, +} from '../lib/flaky.js'; /** * `details` debug block per the CLI OpenAPI `Test` schema @@ -8117,6 +8125,64 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + // ------------------------------------------------------------------------- + // `test flaky` — repeat-run flaky-test detector + // ------------------------------------------------------------------------- + + test + .command('flaky ') + .description( + 'Repeatedly replay a test to measure stability and surface flakiness.\n' + + 'Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail.\n' + + '\nExit codes:\n' + + ' 0 stable (every attempt passed)\n' + + ' 1 flaky or failing (at least one attempt did not pass)\n' + + ' 3 auth error\n' + + ' 4 test not found (no replayable run — trigger `testsprite test run ` first)\n' + + ' 5 validation error', + ) + .option( + '--runs ', + `number of replays to run (1-${MAX_FLAKY_RUNS}, default ${DEFAULT_FLAKY_RUNS})`, + ) + .option( + '--until-fail', + 'stop at the first non-passing attempt (fast "is it flaky at all?" check)', + false, + ) + .option( + '--timeout ', + `per-attempt max seconds to wait (1-${MAX_RUN_TIMEOUT_SECONDS}, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`, + ) + .addHelpText( + 'after', + '\nNotes:\n' + + ' • Frontend replays are free verbatim script replays (no credit); backend replays\n' + + ' re-run the dependency closure and may cost credits — a one-line advisory is printed.\n' + + ' • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass;\n' + + ' this measures replay stability of the saved script against the configured URL.\n' + + ' • `--output json` emits a machine-readable stability report for CI gating.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + testIdArg: string, + cmdOpts: { runs?: string; untilFail?: boolean; timeout?: string }, + command: Command, + ) => { + await runFlaky( + { + ...resolveCommonOptions(command), + testId: testIdArg, + runs: parseNumericFlag(cmdOpts.runs, 'runs') ?? DEFAULT_FLAKY_RUNS, + untilFail: cmdOpts.untilFail === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + }, + ); + test.addCommand(createTestCodeCommand(deps)); test.addCommand(createTestPlanCommand(deps)); test.addCommand(createTestFailureCommand(deps)); @@ -8125,6 +8191,174 @@ export function createTestCommand(deps: TestDeps = {}): Command { return test; } +// --------------------------------------------------------------------------- +// `test flaky` — repeat-run flaky-test detector +// --------------------------------------------------------------------------- + +/** Upper bound on `--runs` so a repeat-runner can't amplify free FE replays. */ +const MAX_FLAKY_RUNS = 10; +/** Default replay count when `--runs` is omitted. */ +const DEFAULT_FLAKY_RUNS = 5; + +interface RunTestFlakyOptions extends CommonOptions { + testId: string; + /** Number of replays to run (1..MAX_FLAKY_RUNS). */ + runs: number; + /** Stop at the first non-passing attempt. */ + untilFail: boolean; + /** Per-attempt polling deadline in seconds. */ + timeoutSeconds: number; +} + +/** + * `test flaky ` — replay a test N times and report a stability score. + * + * Each attempt is a `POST /tests/{id}/runs/rerun` with auto-heal OFF (a strict + * verbatim replay) followed by `pollRunUntilTerminal`. Frontend replays are + * free verbatim script replays; backend replays re-run the dependency closure + * (a one-line credit advisory is printed). The pure scoring lives in + * `lib/flaky.ts`; this function is the I/O orchestrator. + * + * Exit code: 0 when every observed attempt passed (stable), else 1 — so CI can + * gate a merge on flakiness. + */ +export async function runFlaky( + opts: RunTestFlakyOptions, + deps: TestDeps = {}, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + if (typeof opts.testId !== 'string' || opts.testId.length === 0) { + throw localValidationError('test-id', 'is required'); + } + if (!Number.isInteger(opts.runs) || opts.runs < 1 || opts.runs > MAX_FLAKY_RUNS) { + throw localValidationError('runs', `must be an integer between 1 and ${MAX_FLAKY_RUNS}`); + } + + if (opts.dryRun) { + out.print({ + dryRun: true, + command: 'test flaky', + testId: opts.testId, + runs: opts.runs, + untilFail: opts.untilFail, + method: 'POST', + path: `/api/cli/v1/tests/${opts.testId}/runs/rerun`, + note: `Would replay the test up to ${opts.runs}x with auto-heal OFF and report a stability score.`, + }); + return undefined; + } + + // Under the implicit wait, raise the per-request timeout to cover --timeout + // so a slow trigger / long-poll under load isn't cut at the 120s default. + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + + // Best-effort test-type detection for the credit advisory. A probe failure + // never blocks the run — we just skip the advisory. + let isBackend = false; + try { + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + isBackend = test.type === 'backend'; + } catch { + // best-effort — proceed without the advisory. + } + if (isBackend) { + stderrFn( + `[advisory] ${opts.testId} is a backend test — each replay re-runs its dependency closure ` + + `and may cost credits. Frontend replays are free verbatim script replays; backend replays are not.`, + ); + } + + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + const attempts: FlakyAttempt[] = []; + + for (let i = 1; i <= opts.runs; i++) { + const idempotencyKey = `cli-flaky-${randomUUID()}`; + + let rerunResp: RerunResponse; + try { + // auto-heal is intentionally OFF: flaky detection needs a strict verbatim + // replay so healed drift cannot mask a nondeterministic pass/fail. + rerunResp = await client.triggerRerun(opts.testId, { source: 'cli' }, { idempotencyKey }); + } catch (err) { + // A missing replayable run is fatal for the whole command (mirror rerun): + // there is nothing to repeat, so point the user at a fresh `test run`. + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + throw ApiError.fromEnvelope({ + error: { + code: 'NOT_FOUND', + message: `Test ${opts.testId} has no replayable run (unknown/cross-tenant id, or it has never completed a clean run).`, + nextAction: `Trigger a fresh run first: testsprite test run ${opts.testId}`, + requestId: err.requestId ?? 'local', + details: { testId: opts.testId, reason: 'no_replayable_run' }, + }, + }); + } + // Any other trigger error is recorded as an errored attempt so a single + // transient blip doesn't abort a long stability probe. + const code = err instanceof ApiError ? err.code : 'ERROR'; + attempts.push({ attempt: i, runId: null, outcome: 'error', failureKind: code }); + ticker.update(`Attempt ${i}/${opts.runs} — error (${code})`); + if (opts.untilFail) break; + continue; + } + + const runId = rerunResp.runId; + // Backend run rows never finalize server-side; resolve the verdict from the + // testId-scoped result on non-terminal ticks (same fallback as `test rerun`). + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => opts.testId, + resolveNotBefore: () => rerunResp.enqueuedAt, + }); + + let outcome: FlakyOutcome; + let failureKind: string | null = null; + try { + const finalRun = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: opts.timeoutSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + resolveAlternate, + }); + outcome = finalRun.status as FlakyOutcome; + failureKind = finalRun.failureKind; + } catch (err) { + // A per-attempt deadline (poll TimeoutError) or a client-side request + // timeout both count as a non-passing "timeout" outcome for this attempt. + if (err instanceof TimeoutError || err instanceof RequestTimeoutError) { + outcome = 'timeout'; + } else { + throw err; + } + } + + attempts.push({ attempt: i, runId, outcome, failureKind }); + const passedSoFar = attempts.filter(a => a.outcome === 'passed').length; + ticker.update(`Attempt ${i}/${opts.runs} — ${outcome} (${passedSoFar} passed so far)`); + + if (opts.untilFail && outcome !== 'passed') break; + } + + ticker.finalize(); + + const report = summarizeFlaky(opts.testId, attempts); + out.print(report, data => renderFlakyText(data as FlakyReport)); + + const exitCode = flakyExitCode(report); + if (exitCode !== 0) { + throw new CLIError( + `Test ${opts.testId} is ${report.verdict} — ${report.passed}/${report.runs} attempts passed`, + exitCode, + ); + } + return report; +} + interface RunFlagOpts { targetUrl?: string; wait?: boolean; diff --git a/src/lib/flaky.test.ts b/src/lib/flaky.test.ts new file mode 100644 index 0000000..a6acd9c --- /dev/null +++ b/src/lib/flaky.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest'; +import { flakyExitCode, renderFlakyText, summarizeFlaky, type FlakyAttempt } from './flaky.js'; + +function pass(attempt: number): FlakyAttempt { + return { attempt, runId: `run_${attempt}`, outcome: 'passed' }; +} +function fail(attempt: number, failureKind = 'assertion'): FlakyAttempt { + return { attempt, runId: `run_${attempt}`, outcome: 'failed', failureKind }; +} + +describe('summarizeFlaky', () => { + it('reports STABLE when every attempt passed', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2), pass(3)]); + expect(report.verdict).toBe('stable'); + expect(report.runs).toBe(3); + expect(report.passed).toBe(3); + expect(report.failed).toBe(0); + expect(report.stableRatio).toBe(1); + expect(report.failures).toEqual([]); + expect(flakyExitCode(report)).toBe(0); + }); + + it('reports FLAKY on a mix of pass and fail', () => { + const report = summarizeFlaky('test_x', [pass(1), fail(2), pass(3)]); + expect(report.verdict).toBe('flaky'); + expect(report.passed).toBe(2); + expect(report.failed).toBe(1); + expect(report.stableRatio).toBe(0.6667); + expect(report.failures).toEqual([ + { attempt: 2, runId: 'run_2', outcome: 'failed', failureKind: 'assertion' }, + ]); + expect(flakyExitCode(report)).toBe(1); + }); + + it('reports FAILING when no attempt passed', () => { + const report = summarizeFlaky('test_x', [fail(1), fail(2)]); + expect(report.verdict).toBe('failing'); + expect(report.passed).toBe(0); + expect(report.stableRatio).toBe(0); + expect(flakyExitCode(report)).toBe(1); + }); + + it('treats an empty attempt list as FAILING with a 0 ratio', () => { + const report = summarizeFlaky('test_x', []); + expect(report.verdict).toBe('failing'); + expect(report.runs).toBe(0); + expect(report.stableRatio).toBe(0); + expect(flakyExitCode(report)).toBe(1); + }); + + it('counts timeout and error outcomes as non-passing', () => { + const attempts: FlakyAttempt[] = [ + pass(1), + { attempt: 2, runId: 'run_2', outcome: 'timeout' }, + { attempt: 3, runId: null, outcome: 'error', failureKind: 'UNAVAILABLE' }, + ]; + const report = summarizeFlaky('test_x', attempts); + expect(report.verdict).toBe('flaky'); + expect(report.passed).toBe(1); + expect(report.failed).toBe(2); + expect(report.failures.map(f => f.outcome)).toEqual(['timeout', 'error']); + // error attempt with no runId is preserved as null in the report + expect(report.failures[1]).toEqual({ + attempt: 3, + runId: null, + outcome: 'error', + failureKind: 'UNAVAILABLE', + }); + }); + + it('rounds stableRatio to 4 decimal places', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2), fail(3)]); + expect(report.stableRatio).toBe(0.6667); + }); + + it('reflects a short-circuited (--until-fail) run — fewer runs than requested', () => { + // Only two attempts were observed before the probe stopped at the failure. + const report = summarizeFlaky('test_x', [pass(1), fail(2)]); + expect(report.runs).toBe(2); + expect(report.verdict).toBe('flaky'); + }); +}); + +describe('renderFlakyText', () => { + it('summarizes a stable run on one line with no failure list', () => { + const text = renderFlakyText(summarizeFlaky('test_login', [pass(1), pass(2)])); + expect(text).toBe('Ran test_login 2x — 2 passed, 0 failed → STABLE (100% stable)'); + }); + + it('lists failing attempts with runId and failureKind', () => { + const text = renderFlakyText( + summarizeFlaky('test_login', [pass(1), fail(2, 'network_timeout')]), + ); + expect(text).toContain('→ FLAKY (50% stable)'); + expect(text).toContain('failed attempts:'); + expect(text).toContain('#2 run_2 failed failureKind=network_timeout'); + }); + + it('shows (no runId) when an errored attempt never got a runId', () => { + const text = renderFlakyText( + summarizeFlaky('test_login', [{ attempt: 1, runId: null, outcome: 'error' }]), + ); + expect(text).toContain('#1 (no runId) error'); + }); +}); diff --git a/src/lib/flaky.ts b/src/lib/flaky.ts new file mode 100644 index 0000000..402eaf3 --- /dev/null +++ b/src/lib/flaky.ts @@ -0,0 +1,133 @@ +/** + * Pure aggregation + rendering for `test flaky` — the repeat-run flaky-test + * detector. This module performs no I/O: the orchestrator in + * `commands/test.ts` feeds it the per-attempt outcomes, and it returns a + * machine-readable stability report plus the text rendering and exit code. + * + * Keeping the scoring logic pure makes it trivially unit-testable in isolation + * (deterministic, no network / credentials), matching the repo's mock-based + * test convention. + */ + +/** + * Outcome of a single flaky-detector attempt. The first four mirror the + * terminal `RunStatus` values; `timeout` and `error` are orchestration + * outcomes (per-attempt deadline exceeded, or a non-fatal trigger/transport + * error that was recorded rather than aborting the whole probe). + */ +export type FlakyOutcome = 'passed' | 'failed' | 'blocked' | 'cancelled' | 'timeout' | 'error'; + +/** Overall stability verdict across all observed attempts. */ +export type FlakyVerdict = 'stable' | 'flaky' | 'failing'; + +/** One recorded attempt in a flaky run. */ +export interface FlakyAttempt { + /** 1-based attempt index. */ + attempt: number; + /** The runId of this attempt, or `null` when the trigger never returned one. */ + runId: string | null; + outcome: FlakyOutcome; + /** + * Server `failureKind` for non-passing runs (or the error code for `error` + * outcomes). `null` / omitted when the attempt passed or the kind is unknown. + */ + failureKind?: string | null; +} + +/** A single non-passing attempt as surfaced in the report. */ +export interface FlakyFailure { + attempt: number; + runId: string | null; + outcome: FlakyOutcome; + failureKind: string | null; +} + +/** + * Machine-readable stability report. This is also the exact `--output json` + * shape, so dashboards / agents / CI can consume it directly. + */ +export interface FlakyReport { + testId: string; + /** + * Attempts actually observed. May be fewer than requested when + * `--until-fail` short-circuits on the first non-passing attempt. + */ + runs: number; + passed: number; + failed: number; + /** `passed / runs`, rounded to 4 decimal places. `0` when `runs === 0`. */ + stableRatio: number; + verdict: FlakyVerdict; + /** Non-passing attempts, in attempt order. */ + failures: FlakyFailure[]; +} + +/** + * Aggregate per-attempt outcomes into a stability report. + * + * Verdict rules: + * - every attempt passed → `stable` + * - no attempt passed → `failing` + * - a mix of pass and non-pass → `flaky` + * An empty attempt list (no runs observed) is reported as `failing` with a + * `0` ratio — there is no evidence the test is stable. + */ +export function summarizeFlaky(testId: string, attempts: FlakyAttempt[]): FlakyReport { + const runs = attempts.length; + const passed = attempts.filter(a => a.outcome === 'passed').length; + const failed = runs - passed; + const stableRatio = runs === 0 ? 0 : Math.round((passed / runs) * 10000) / 10000; + const verdict: FlakyVerdict = + runs > 0 && passed === runs ? 'stable' : passed === 0 ? 'failing' : 'flaky'; + const failures: FlakyFailure[] = attempts + .filter(a => a.outcome !== 'passed') + .map(a => ({ + attempt: a.attempt, + runId: a.runId, + outcome: a.outcome, + failureKind: a.failureKind ?? null, + })); + return { testId, runs, passed, failed, stableRatio, verdict, failures }; +} + +/** + * Exit code for the command: `0` only when the verdict is `stable` (every + * observed attempt passed). Anything else is non-zero so CI can gate a merge + * on flakiness (`testsprite test flaky --runs 5 || exit 1`). + */ +export function flakyExitCode(report: FlakyReport): number { + return report.verdict === 'stable' ? 0 : 1; +} + +/** Human-readable label for a verdict. */ +function verdictLabel(verdict: FlakyVerdict): string { + switch (verdict) { + case 'stable': + return 'STABLE'; + case 'flaky': + return 'FLAKY'; + case 'failing': + return 'FAILING'; + } +} + +/** + * Render a report to human-readable text. JSON-mode callers ship the report + * verbatim via `out.print`; this is the text-mode rendering. + */ +export function renderFlakyText(report: FlakyReport): string { + const pct = Math.round(report.stableRatio * 100); + const lines: string[] = [ + `Ran ${report.testId} ${report.runs}x — ${report.passed} passed, ${report.failed} failed → ` + + `${verdictLabel(report.verdict)} (${pct}% stable)`, + ]; + if (report.failures.length > 0) { + lines.push(' failed attempts:'); + for (const f of report.failures) { + const kind = f.failureKind ? ` failureKind=${f.failureKind}` : ''; + const rid = f.runId ?? '(no runId)'; + lines.push(` #${f.attempt} ${rid} ${f.outcome}${kind}`); + } + } + return lines.join('\n'); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index d89e7dc..39e57dd 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -270,6 +270,15 @@ Commands: 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get + flaky [options] Repeatedly replay a test to measure stability and surface flakiness. + Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. + + Exit codes: + 0 stable (every attempt passed) + 1 flaky or failing (at least one attempt did not pass) + 3 auth error + 4 test not found (no replayable run — trigger \`testsprite test run \` first) + 5 validation error code Inspect and edit generated test code plan Manage FE test plan-steps (FE-only) failure Export the latest-failure agent bundle @@ -345,6 +354,38 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeou " `; +exports[`--help snapshots > test flaky 1`] = ` +"Usage: testsprite test flaky [options] + +Repeatedly replay a test to measure stability and surface flakiness. +Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. + +Exit codes: + 0 stable (every attempt passed) + 1 flaky or failing (at least one attempt did not pass) + 3 auth error + 4 test not found (no replayable run — trigger \`testsprite test run \` first) + 5 validation error + +Options: + --runs number of replays to run (1-10, default 5) + --until-fail stop at the first non-passing attempt (fast "is it flaky at + all?" check) (default: false) + --timeout per-attempt max seconds to wait (1-3600, default 600) + -h, --help display help for command + +Notes: + • Frontend replays are free verbatim script replays (no credit); backend replays + re-run the dependency closure and may cost credits — a one-line advisory is printed. + • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass; + this measures replay stability of the saved script against the configured URL. + • \`--output json\` emits a machine-readable stability report for CI gating. + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test get 1`] = ` "Usage: testsprite test get [options] diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index 2448c91..ee89b3b 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -39,6 +39,7 @@ const cases: Array<[string, string[]]> = [ ['test result', ['test', 'result', '--help']], ['test failure get', ['test', 'failure', 'get', '--help']], ['test rerun', ['test', 'rerun', '--help']], + ['test flaky', ['test', 'flaky', '--help']], // R5: regression guard for commands that gained new flag wording ['test create-batch', ['test', 'create-batch', '--help']], ['test run', ['test', 'run', '--help']], From b9e9601c4926e21c89066d30d56312524036ecb0 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:28:01 -0400 Subject: [PATCH 052/117] feat(test): add "test lint" offline plan/steps validator (#176) * feat(test): add "test lint" offline plan/steps validator * fix(lint): report physical JSONL line numbers (blank lines no longer shift them) --- src/commands/test.test.ts | 81 +++++++++ src/commands/test.ts | 169 ++++++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 5 + 3 files changed, 255 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 6d8d734..c78c7a8 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -31,6 +31,7 @@ import { runFailureGet, runFailureSummary, runGet, + runLint, runList, runPlanPut, runResult, @@ -126,6 +127,7 @@ describe('createTestCommand — surface', () => { 'failure', 'flaky', 'get', + 'lint', 'list', 'plan', 'rerun', @@ -2961,6 +2963,85 @@ describe('runDiff', () => { }); }); +describe('runLint', () => { + const VALID_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Checkout works', + planSteps: [ + { type: 'action', description: 'Open the cart' }, + { type: 'assertion', description: 'Assert the total is visible' }, + ], + }); + const INVALID_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Broken', + planSteps: [{ type: 'hover', description: 'Bad step type' }], + }); + + it('a directory with valid and invalid plans reports EVERY problem and exits 5', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-')); + writeFileSync(join(dir, 'a-valid.json'), VALID_PLAN, 'utf8'); + writeFileSync(join(dir, 'b-invalid.json'), INVALID_PLAN, 'utf8'); + writeFileSync(join(dir, 'c-notjson.json'), '{oops', 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + checked: number; + valid: number; + issues: Array<{ file: string; field: string }>; + }; + // All three files were checked (no first-error-fatal bailout). + expect(report.checked).toBe(3); + expect(report.valid).toBe(1); + expect(report.issues.map(issue => issue.file).sort()).toEqual([ + 'b-invalid.json', + 'c-notjson.json', + ]); + }); + + it('an all-valid directory resolves with exit 0 and no network/credentials needed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-ok-')); + writeFileSync(join(dir, 'a.json'), VALID_PLAN, 'utf8'); + const report = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: () => undefined }, + ); + expect(report).toMatchObject({ checked: 1, valid: 1, issues: [] }); + }); + + it('a JSONL file reports each bad line with its PHYSICAL line number (blank lines skipped, not renumbered)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-jsonl-')); + const file = join(dir, 'plans.jsonl'); + // A blank separator line sits between entries: line 1 valid, line 2 blank, + // line 3 bad JSON, line 4 invalid plan. Reported numbers must be 3 and 4. + writeFileSync(file, `${VALID_PLAN}\n\nnot json at all\n${INVALID_PLAN}\n`, 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, plans: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { checked: number; issues: Array<{ file: string }> }; + expect(report.checked).toBe(3); + expect(report.issues.some(issue => issue.file.endsWith(':3'))).toBe(true); + expect(report.issues.some(issue => issue.file.endsWith(':4'))).toBe(true); + // The blank line itself is not an entry and never reports. + expect(report.issues.some(issue => issue.file.endsWith(':2'))).toBe(false); + }); + + it('requires exactly one input source', async () => { + await expect( + runLint({ profile: 'default', output: 'json', debug: false }, { stdout: () => undefined }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + describe('runResult', () => { it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index 508d8be..f11e688 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3862,6 +3862,144 @@ function renderRunDiffText(diff: CliRunDiff): string { return lines.join('\n'); } +export interface LintOptions extends CommonOptions { + planFrom?: string; + planFromDir?: string; + plans?: string; + steps?: string; +} + +export interface CliLintIssue { + file: string; + field: string; + reason: string; +} + +export interface CliLintReport { + checked: number; + valid: number; + issues: CliLintIssue[]; +} + +/** + * `test lint` (issue #98): validate plan/steps files fully OFFLINE with the + * SAME validators the create paths run, but collecting EVERY problem instead + * of dying on the first one, and without any network write. The create-batch + * reader is first-error-fatal and only reachable through a command that POSTs, + * so authoring a 12-plan directory meant one error per paid round-trip. Zero + * network, zero credentials: exit 0 when everything is valid, 5 otherwise, so + * it drops into a pre-commit hook or CI step before `create-batch`. + */ +export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const sources = [opts.planFrom, opts.planFromDir, opts.plans, opts.steps].filter( + source => source !== undefined, + ); + if (sources.length !== 1) { + throw localValidationError( + 'plan-from', + 'exactly one of --plan-from, --plan-from-dir, --plans, or --steps is required', + ); + } + + const issues: CliLintIssue[] = []; + let checked = 0; + // Run one existing validator, converting its typed throw into a report row + // (same envelopes, so `details.field` pointers like planSteps[2].type + // survive verbatim). + const collect = (file: string, validate: () => void): void => { + checked += 1; + try { + validate(); + } catch (err) { + if (err instanceof ApiError) { + issues.push({ + file, + field: String(err.getDetail('field') ?? '(file)'), + reason: String(err.getDetail('reason') ?? err.nextAction ?? err.message), + }); + } else { + issues.push({ + file, + field: '(file)', + reason: err instanceof Error ? err.message : String(err), + }); + } + } + }; + + if (opts.planFrom !== undefined) { + const planFrom = opts.planFrom; + collect(planFrom, () => void readPlanFromGuarded(planFrom)); + } else if (opts.steps !== undefined) { + const steps = opts.steps; + collect(steps, () => void readPlanStepsFileGuarded(steps)); + } else if (opts.planFromDir !== undefined) { + const dir = resolveAbsolute(opts.planFromDir); + let entries: string[]; + try { + entries = readdirSync(dir) + .filter(name => name.endsWith('.json')) + .sort(); + } catch { + throw localValidationError('plan-from-dir', `cannot read directory: ${dir}`); + } + if (entries.length === 0) { + throw localValidationError('plan-from-dir', 'contains no *.json plan files'); + } + for (const entry of entries) { + collect(entry, () => void readPlanFromGuarded(join(dir, entry))); + } + } else if (opts.plans !== undefined) { + // JSONL: validate PER LINE so every bad line reports (the create path's + // reader stays throw-on-first; this is the collecting counterpart). + const absolute = resolveAbsolute(opts.plans); + let content: string; + try { + content = readFileSync(absolute, 'utf8'); + } catch { + throw localValidationError('plans', `cannot read file: ${absolute}`); + } + // Index lines BEFORE dropping blanks so every reported `file:N` points at + // the PHYSICAL line in the file (a blank separator line must not shift all + // subsequent line numbers). + const numberedLines = content + .split('\n') + .map((rawLine, physicalIndex) => ({ line: rawLine.trim(), lineNo: physicalIndex + 1 })) + .filter(entry => entry.line.length > 0); + if (numberedLines.length === 0) throw localValidationError('plans', 'contains no plan lines'); + for (const { line, lineNo } of numberedLines) { + collect(`${opts.plans}:${lineNo}`, () => { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + throw localValidationError( + 'plans', + `line ${lineNo} is not valid JSON`, + undefined, + 'field', + ); + } + assertPlanShape(parsed, { specIndex: lineNo - 1 }); + }); + } + } + + const filesWithIssues = new Set(issues.map(issue => issue.file)).size; + const report: CliLintReport = { checked, valid: checked - filesWithIssues, issues }; + out.print(report, () => + [ + ...issues.map(issue => `${issue.file}: ${issue.field}: ${issue.reason}`), + `${report.valid}/${report.checked} valid, ${issues.length} problem(s)`, + ].join('\n'), + ); + if (issues.length > 0) { + throw new CLIError(`lint: ${issues.length} problem(s) across ${report.checked} file(s)`, 5); + } + return report; +} + export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -7669,6 +7807,37 @@ export function createTestCommand(deps: TestDeps = {}): Command { await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps); }); + test + .command('lint') + .description( + 'Validate plan/steps files offline with the same validators `create` runs, collecting EVERY problem. No network, no credentials. Exit 0 when all valid, 5 otherwise.', + ) + .option('--plan-from ', 'single plan JSON file') + .option( + '--plan-from-dir ', + 'directory of *.json plan files (each checked, all errors reported)', + ) + .option('--plans ', 'JSONL file with one plan spec per line (each line checked)') + .option('--steps ', 'plan-steps JSON file (the shape `test plan put` ingests)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + cmdOpts: { planFrom?: string; planFromDir?: string; plans?: string; steps?: string }, + command: Command, + ) => { + await runLint( + { + ...resolveCommonOptions(command), + planFrom: cmdOpts.planFrom, + planFromDir: cmdOpts.planFromDir, + plans: cmdOpts.plans, + steps: cmdOpts.steps, + }, + deps, + ); + }, + ); + test .command('result ') .description( diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 39e57dd..9fb6b2d 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -208,6 +208,11 @@ Commands: failedStepIndex, per-step status flips, codeVersion drift. Exit 0 when verdicts match, 1 when they differ. + lint [options] Validate plan/steps files offline with + the same validators \`create\` runs, + collecting EVERY problem. No network, + no credentials. Exit 0 when all valid, + 5 otherwise. result [options] Get the latest result for a test (default) or list prior runs (--history). --output json shape differs by mode: From f7b10b724442b81b46275a51b4f0a37051ba19af Mon Sep 17 00:00:00 2001 From: Sahil Rakhaiya <144577420+SahilRakhaiya05@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:39:51 +0530 Subject: [PATCH 053/117] fix(config): treat empty env vars as unset in loadConfig (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(config): treat empty env vars as unset in loadConfig * test: address review — dry-run whoami uses offline client; fix whitespace env key expectation --- src/commands/auth.test.ts | 14 ++++++++++++++ src/commands/auth.ts | 7 ++++--- src/commands/init.ts | 3 ++- src/lib/client-factory.test.ts | 21 ++++++++++++++++++++- src/lib/config.test.ts | 22 ++++++++++++++++++++++ src/lib/config.ts | 15 +++++++++++++-- 6 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 1605f58..c0b4e00 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -850,6 +850,20 @@ describe('runWhoami', () => { expect(printed).toEqual(sampleMe); }); + it('dry-run: whitespace-only TESTSPRITE_API_URL falls through to prod default endpoint', async () => { + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + { + ...deps, + env: { TESTSPRITE_API_URL: ' ' }, + credentialsPath, + }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('endpoint: https://api.testsprite.com'); + }); + it('L1788: text output includes the resolved endpoint URL', async () => { writeProfile( 'default', diff --git a/src/commands/auth.ts b/src/commands/auth.ts index bf2eb88..59cd43f 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -17,7 +17,7 @@ import { readProfile, writeProfile, } from '../lib/credentials.js'; -import { loadConfig } from '../lib/config.js'; +import { loadConfig, normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; @@ -85,7 +85,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // treated as unset. Without this, `''` (e.g. `export TESTSPRITE_API_URL=` in a // shell profile) is non-nullish and would short-circuit the `??` chains below to // an empty endpoint instead of falling through to the profile / prod default. - const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); // Dry-run: do not prompt, do not read env, do not write credentials. // Print the canned success shape so an agent sees exactly the JSON it @@ -208,7 +208,8 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi // displayed URL always matches where requests actually go (dogfood L1788). let resolvedEndpoint: string; if (opts.dryRun) { - resolvedEndpoint = opts.endpointUrl ?? env.TESTSPRITE_API_URL ?? 'https://api.testsprite.com'; + resolvedEndpoint = + opts.endpointUrl ?? normalizeEnvVar(env.TESTSPRITE_API_URL) ?? 'https://api.testsprite.com'; } else { const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const config = loadConfig({ diff --git a/src/commands/init.ts b/src/commands/init.ts index e01215a..aff0644 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -15,6 +15,7 @@ import { parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; +import { normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; @@ -39,7 +40,7 @@ const DEFAULT_API_URL = 'https://api.testsprite.com'; */ function resolveReportedEndpoint(opts: InitOptions, deps: InitDeps): string { const env = deps.env ?? process.env; - const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); let existing: string | undefined; try { existing = readProfile(opts.profile, { path: deps.credentialsPath })?.apiUrl; diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index ab5fd99..d5135cc 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -339,7 +339,6 @@ describe('makeHttpClient - API key validation', () => { ['smart dash', 'sk-user-abc\u2013def'], ['smart quote', 'sk-user-\u201cabc\u201d'], ['emoji', 'sk-user-abc\u{1f600}'], - ['whitespace-only', ' '], ])('rejects a malformed configured API key with %s before fetch/retry', (_label, apiKey) => { const fetchImpl = vi.fn(); let caught: unknown; @@ -362,6 +361,26 @@ describe('makeHttpClient - API key validation', () => { expect(apiErr.exitCode).toBe(5); expect(apiErr.nextAction).toContain('api-key'); }); + + it('treats a whitespace-only TESTSPRITE_API_KEY env var as unset (AUTH_REQUIRED)', () => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: ' ' } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + expect((caught as ApiError).code).toBe('AUTH_REQUIRED'); + }); }); describe('assertValidApiKeyHeaderValue', () => { diff --git a/src/lib/config.test.ts b/src/lib/config.test.ts index 463ddcc..cf56eaf 100644 --- a/src/lib/config.test.ts +++ b/src/lib/config.test.ts @@ -81,6 +81,28 @@ describe('loadConfig', () => { const config = loadConfig({ profile: 'dev', env: {}, credentialsPath }); expect(config.apiKey).toBe('sk-dev'); }); + + it('treats empty / whitespace TESTSPRITE_API_URL as unset (falls through to profile)', () => { + writeProfile( + 'default', + { apiKey: 'sk-file', apiUrl: 'https://api.example.com:8443' }, + { path: credentialsPath }, + ); + const config = loadConfig({ + env: { TESTSPRITE_API_URL: ' ' }, + credentialsPath, + }); + expect(config.apiUrl).toBe('https://api.example.com:8443'); + }); + + it('treats empty / whitespace TESTSPRITE_API_KEY as unset (falls through to profile)', () => { + writeProfile('default', { apiKey: 'sk-file' }, { path: credentialsPath }); + const config = loadConfig({ + env: { TESTSPRITE_API_KEY: '' }, + credentialsPath, + }); + expect(config.apiKey).toBe('sk-file'); + }); }); describe('defaultConfigPath', () => { diff --git a/src/lib/config.ts b/src/lib/config.ts index 363c394..7f8065f 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -17,6 +17,11 @@ export interface LoadConfigOptions { const DEFAULT_API_URL = 'https://api.testsprite.com'; +/** Treat empty / whitespace-only env values as unset for `??` resolution chains. */ +export function normalizeEnvVar(value: string | undefined): string | undefined { + return value?.trim() || undefined; +} + export function defaultConfigPath(): string { return join(homedir(), '.testsprite', 'config'); } @@ -38,9 +43,15 @@ export function loadConfig(options: LoadConfigOptions = {}): Config { const credentialsPath = options.credentialsPath ?? defaultCredentialsPath(); const fileEntry = readProfile(profile, { path: credentialsPath }); + // Empty / whitespace-only env vars are treated as unset so they do not + // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` in a shell + // profile). Matches the normalization in auth configure and init/setup. + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); + const envApiKey = normalizeEnvVar(env.TESTSPRITE_API_KEY); + return { - apiUrl: options.endpointUrl ?? env.TESTSPRITE_API_URL ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, - apiKey: env.TESTSPRITE_API_KEY ?? fileEntry?.apiKey, + apiUrl: options.endpointUrl ?? envApiUrl ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, + apiKey: envApiKey ?? fileEntry?.apiKey, profile, }; } From c6d37b7e5cfc85cd56b3ac2f5d39c7177e019a46 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:10:25 -0500 Subject: [PATCH 054/117] fix(http): clear per-attempt timeout timers (#137) * fix(http): clear per-attempt timeout timers * fix(http): preserve timeout race classification --- src/lib/http.test.ts | 73 +++++++++ src/lib/http.ts | 359 ++++++++++++++++++++++++------------------- 2 files changed, 277 insertions(+), 155 deletions(-) diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 0320367..a9c1193 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -449,6 +449,52 @@ describe('HttpClient per-request timeout', () => { expect(callCount).toBe(1); }); + it('clears the per-attempt timeout after a successful response body is read', async () => { + let capturedSignal: AbortSignal | undefined; + const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { + capturedSignal = init?.signal ?? undefined; + return jsonResponse({ ok: true }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => Promise.resolve(), + random: () => 0, + requestTimeoutMs: 25, + }); + + await expect(client.get('/me')).resolves.toEqual({ ok: true }); + expect(capturedSignal?.aborted).toBe(false); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(capturedSignal?.aborted).toBe(false); + }); + + it('clears a failed attempt timeout before retry backoff sleeps', async () => { + const attemptSignals: AbortSignal[] = []; + let calls = 0; + const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { + if (init?.signal) attemptSignals.push(init.signal); + calls += 1; + if (calls === 1) return errorEnvelopeResponse(500, 'INTERNAL'); + return jsonResponse({ ok: true }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => new Promise(resolve => setTimeout(resolve, 50)), + random: () => 0, + requestTimeoutMs: 25, + }); + + await expect(client.get('/me')).resolves.toEqual({ ok: true }); + expect(attemptSignals).toHaveLength(2); + expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); + }); + it('caller-supplied AbortSignal still propagates as AbortError (not RequestTimeoutError)', async () => { const controller = new AbortController(); // Abort immediately @@ -475,6 +521,33 @@ describe('HttpClient per-request timeout', () => { expect((err as Error).name).toBe('AbortError'); }); + it('preserves RequestTimeoutError when the caller aborts after the request timeout wins', async () => { + const controller = new AbortController(); + const fetchImpl = vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + const reason = init.signal?.reason; + controller.abort(new Error('caller aborted after timeout')); + const err = new Error(reason?.message ?? 'timed out'); + err.name = reason?.name ?? 'TimeoutError'; + reject(err); + }); + }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => Promise.resolve(), + random: () => 0, + requestTimeoutMs: 1, + }); + const err = await client.get('/me', { signal: controller.signal }).catch(e => e); + expect(err).toBeInstanceOf(RequestTimeoutError); + expect((err as RequestTimeoutError).exitCode).toBe(7); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + it('defaults to REQUEST_TIMEOUT_DEFAULT_MS when no requestTimeoutMs is supplied', () => { // Verify the default is wired without actually waiting 120s. // We test via the exported constant rather than exercising the stall. diff --git a/src/lib/http.ts b/src/lib/http.ts index b5dabb1..af1c25a 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -395,9 +395,15 @@ export class HttpClient { timeoutSignal: AbortSignal, callerSignal: AbortSignal | undefined, requestId: string, + effectiveSignal: AbortSignal = timeoutSignal, ): void { if (isAbortError(err) || isTimeoutError(err)) { - if (timeoutSignal.aborted && (callerSignal == null || !callerSignal.aborted)) { + const timeoutWon = + timeoutSignal.aborted && + (callerSignal == null || + !callerSignal.aborted || + effectiveSignal.reason === timeoutSignal.reason); + if (timeoutWon) { throw new RequestTimeoutError(this.requestTimeoutMs, requestId); } throw err; @@ -428,190 +434,200 @@ export class HttpClient { // signal. The polling path supplies its own deadline-aware signal per // iteration — this timeout (120s default) is safely larger than any single // long-poll window (<=25s via ?waitSeconds), so it never bites polling. - const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs); + const requestTimeout = createRequestTimeout(this.requestTimeoutMs); + const timeoutSignal = requestTimeout.signal; const effectiveSignal = options.signal != null ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal; try { - response = await this.fetchImpl(url, { - method, - headers: this.buildHeaders(requestId, options), - body: options.body !== undefined ? JSON.stringify(options.body) : undefined, - signal: effectiveSignal, - }); - } catch (err) { - // Distinguish a client-side request timeout from a caller-supplied abort. - // - // Node 22 `AbortSignal.timeout()` throws a `DOMException` with - // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. - // A caller-supplied abort sets `name === 'AbortError'`. - // We treat both abort variants together: if the timeout signal fired and - // the caller hadn't already aborted, surface a clear RequestTimeoutError. - // A timeout/abort during the fetch itself: classify it (RequestTimeoutError - // when our deadline fired; otherwise rethrow the caller's abort unmodified). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - // If a RequestTimeoutError already propagated from somewhere (e.g. from a - // nested call or from a test-injected fetchImpl), pass it through unchanged - // rather than re-wrapping it as a TransportError. - if (err instanceof RequestTimeoutError) throw err; - const message = err instanceof Error ? err.message : String(err); - this.debug({ - kind: 'error', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - durationMs: Date.now() - startedAt, - }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) throw new TransportError(message, requestId); - this.transition( - `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, - ); - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - delayMs: decision.delayMs, - }); - await this.sleep(decision.delayMs); - continue; - } + try { + response = await this.fetchImpl(url, { + method, + headers: this.buildHeaders(requestId, options), + body: options.body !== undefined ? JSON.stringify(options.body) : undefined, + signal: effectiveSignal, + }); + } catch (err) { + // Distinguish a client-side request timeout from a caller-supplied abort. + // + // The request-timeout controller aborts with an Error/DOMException whose + // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. + // A caller-supplied abort sets `name === 'AbortError'`. + // We treat both abort variants together: if the timeout signal fired and + // the caller hadn't already aborted, surface a clear RequestTimeoutError. + // A timeout/abort during the fetch itself: classify it (RequestTimeoutError + // when our deadline fired; otherwise rethrow the caller's abort unmodified). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + // If a RequestTimeoutError already propagated from somewhere (e.g. from a + // nested call or from a test-injected fetchImpl), pass it through unchanged + // rather than re-wrapping it as a TransportError. + if (err instanceof RequestTimeoutError) throw err; + const message = err instanceof Error ? err.message : String(err); + this.debug({ + kind: 'error', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + durationMs: Date.now() - startedAt, + }); + const decision = transportRetryDecision(attempt, this.random); + if (!decision.retry) throw new TransportError(message, requestId); + this.transition( + `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + requestTimeout.clear(); + await this.sleep(decision.delayMs); + continue; + } - const durationMs = Date.now() - startedAt; - if (response.ok) { - this.debug({ - kind: 'response', - method, - url, - attempt, - status: response.status, - requestId, - durationMs, - }); + const durationMs = Date.now() - startedAt; + if (response.ok) { + this.debug({ + kind: 'response', + method, + url, + attempt, + status: response.status, + requestId, + durationMs, + }); + try { + return { body: (await response.json()) as T, requestId, status: response.status }; + } catch (err) { + // A timeout/abort can fire mid-body-read (headers received, stream stalls). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + // Otherwise the successful response body was not valid JSON — a + // misconfigured endpoint, a proxy / captive-portal / login page that + // returns HTML with a 200 status, or an empty body. Surface a typed + // error carrying the requestId instead of letting the raw SyntaxError + // escape to index.ts, where it would print a bare `{"error":"..."}` + // and break the --output json envelope contract. + throw malformedResponseError(response, requestId, err); + } + } + + let rawBody: unknown; try { - return { body: (await response.json()) as T, requestId, status: response.status }; + rawBody = await safeReadJson(response); } catch (err) { - // A timeout/abort can fire mid-body-read (headers received, stream stalls). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - // Otherwise the successful response body was not valid JSON — a - // misconfigured endpoint, a proxy / captive-portal / login page that - // returns HTML with a 200 status, or an empty body. Surface a typed - // error carrying the requestId instead of letting the raw SyntaxError - // escape to index.ts, where it would print a bare `{"error":"..."}` - // and break the --output json envelope contract. - throw malformedResponseError(response, requestId, err); + // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse + // errors), so a timeout fired mid-body-read on a non-OK response lands here. + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + throw err; } - } - let rawBody: unknown; - try { - rawBody = await safeReadJson(response); - } catch (err) { - // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse - // errors), so a timeout fired mid-body-read on a non-OK response lands here. - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - throw err; - } + // Edge proxies / load balancers return 408/502/504 without our error + // envelope on transient outages. Per the CLI error spec §7 these are + // transport-level retries, not facade errors — fold them in here so + // we get the bounded backoff budget instead of a single INTERNAL bail. + if (rawBody === null && isTransportEdgeStatus(response.status)) { + this.debug({ + kind: 'error', + method, + url, + attempt, + status: response.status, + requestId, + errorCode: 'TRANSPORT', + durationMs, + }); + const decision = transportRetryDecision(attempt, this.random); + if (!decision.retry) { + throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); + } + this.transition( + `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + requestTimeout.clear(); + await this.sleep(decision.delayMs); + continue; + } - // Edge proxies / load balancers return 408/502/504 without our error - // envelope on transient outages. Per the CLI error spec §7 these are - // transport-level retries, not facade errors — fold them in here so - // we get the bounded backoff budget instead of a single INTERNAL bail. - if (rawBody === null && isTransportEdgeStatus(response.status)) { + const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); + // Clamp server-directed Retry-After to [1s, 300s] and surface on the + // thrown error so outer callers (e.g. runBatchRun outer retry loop) + // can honor it without re-reading the now-consumed HTTP response. + const retryAfterMsForError = + retryAfterSec !== undefined + ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 + : undefined; + const apiError = ApiError.fromEnvelope( + rawBody, + response.status, + retryAfterMsForError, + // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing + // links) resolve the environment-correct portal domain. + this.baseUrl, + ); this.debug({ kind: 'error', method, url, attempt, status: response.status, + errorCode: apiError.code, requestId, - errorCode: 'TRANSPORT', durationMs, }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) { - throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); - } - this.transition( - `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + const retryOnConflict = options.retryOnConflict !== false; + const retryOnRateLimit = options.retryOnRateLimit !== false; + const decision = apiRetryDecision( + apiError.code, + attempt, + retryAfterSec, + this.random, + retryOnConflict, + retryOnRateLimit, ); + if (!decision.retry) throw apiError; + const delaySec = Math.round(decision.delayMs / 1000); + if (apiError.code === 'RATE_LIMITED') { + this.transition( + `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, + ); + } else if (apiError.code === 'INTERNAL') { + this.transition( + `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } else if (apiError.code === 'UNAVAILABLE') { + this.transition( + `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } this.debug({ kind: 'retry', method, url, attempt, requestId, - errorCode: 'TRANSPORT', + errorCode: apiError.code, delayMs: decision.delayMs, }); + requestTimeout.clear(); await this.sleep(decision.delayMs); - continue; + } finally { + requestTimeout.clear(); } - - const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); - // Clamp server-directed Retry-After to [1s, 300s] and surface on the - // thrown error so outer callers (e.g. runBatchRun outer retry loop) - // can honor it without re-reading the now-consumed HTTP response. - const retryAfterMsForError = - retryAfterSec !== undefined ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 : undefined; - const apiError = ApiError.fromEnvelope( - rawBody, - response.status, - retryAfterMsForError, - // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing - // links) resolve the environment-correct portal domain. - this.baseUrl, - ); - this.debug({ - kind: 'error', - method, - url, - attempt, - status: response.status, - errorCode: apiError.code, - requestId, - durationMs, - }); - const retryOnConflict = options.retryOnConflict !== false; - const retryOnRateLimit = options.retryOnRateLimit !== false; - const decision = apiRetryDecision( - apiError.code, - attempt, - retryAfterSec, - this.random, - retryOnConflict, - retryOnRateLimit, - ); - if (!decision.retry) throw apiError; - const delaySec = Math.round(decision.delayMs / 1000); - if (apiError.code === 'RATE_LIMITED') { - this.transition( - `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, - ); - } else if (apiError.code === 'INTERNAL') { - this.transition( - `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, - ); - } else if (apiError.code === 'UNAVAILABLE') { - this.transition( - `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, - ); - } - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: apiError.code, - delayMs: decision.delayMs, - }); - await this.sleep(decision.delayMs); } } @@ -687,6 +703,39 @@ function newRequestId(): string { return `cli_${randomUUID()}`; } +interface RequestTimeoutHandle { + signal: AbortSignal; + clear: () => void; +} + +function createRequestTimeout(timeoutMs: number): RequestTimeoutHandle { + const controller = new AbortController(); + const timer = setTimeout(() => { + controller.abort(makeTimeoutReason()); + }, timeoutMs); + unrefTimer(timer); + + return { + signal: controller.signal, + clear: () => clearTimeout(timer), + }; +} + +function makeTimeoutReason(): Error { + if (typeof DOMException !== 'undefined') { + return new DOMException('The operation timed out.', 'TimeoutError'); + } + const err = new Error('The operation timed out.'); + err.name = 'TimeoutError'; + return err; +} + +function unrefTimer(timer: ReturnType): void { + if (typeof timer !== 'object' || timer === null || !('unref' in timer)) return; + const unref = (timer as { unref?: () => void }).unref; + if (typeof unref === 'function') unref.call(timer); +} + async function safeReadJson(response: Response): Promise { try { return await response.json(); From 78ad224ba55231d383f7bc44f949b8ad016235d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:10:58 -0700 Subject: [PATCH 055/117] chore(ci): bump marocchino/sticky-pull-request-comment (#187) Bumps [marocchino/sticky-pull-request-comment](https://github.com/marocchino/sticky-pull-request-comment) from 2.9.4 to 3.0.5. - [Release notes](https://github.com/marocchino/sticky-pull-request-comment/releases) - [Commits](https://github.com/marocchino/sticky-pull-request-comment/compare/773744901bac0e8cbb5a0dc842800d45e9b2b405...5770ad5eb8f42dd2c4f34da00c94c5381e49af88) --- updated-dependencies: - dependency-name: marocchino/sticky-pull-request-comment dependency-version: 3.0.5 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 4eb15a9..ebb3331 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -73,7 +73,7 @@ jobs: fi - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 if: github.event_name == 'pull_request' && steps.coverage-summary.outputs.coverage_generated == 'true' with: recreate: true From 108a91315762ccb8c1c157cbc5e15fb1914a7401 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Mon, 6 Jul 2026 23:11:31 +0300 Subject: [PATCH 056/117] fix(batch): classify RequestTimeoutError as timeout in --all --wait fan-out (#154) * fix(batch): classify RequestTimeoutError as timeout in --all --wait fan-out When test run --all --wait or test rerun --all --wait hit a per-request timeout during batch fan-out polling, RequestTimeoutError rejected the fan-out before out.print(). Classify it as status:'timeout' in pollFreshAccepted and pollAccepted so stdout always lists every dispatched runId. Co-authored-by: Cursor * Remove unused readFileSync import from test file --------- Co-authored-by: Contributor Co-authored-by: Cursor --- src/commands/test.rerun.spec.ts | 471 ++------------------------------ src/commands/test.run.spec.ts | 226 +++------------ src/commands/test.ts | 32 +++ 3 files changed, 90 insertions(+), 639 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 129d859..9f16c4b 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4820,326 +4820,38 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { }); // --------------------------------------------------------------------------- -// [fix-exitcode] pollAccepted preserves ApiError exit codes (not hardcoded 1) +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- -describe('[fix-exitcode] polling error exit codes preserved in batch rerun results', () => { - it('AUTH_REQUIRED during polling → batch escalates to exitCode 3', async () => { +describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { const creds = makeCreds(); - const passRun = makeTerminalRun('run_pass_a1', 'passed'); const batchResp: BatchRerunResponse = { accepted: [ - { testId: 'test_1', runId: 'run_auth_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_pass_a1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_auth_fail')) return errorBody('AUTH_REQUIRED'); - if (url.includes('/runs/run_pass_a1')) return { body: passRun }; - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e as { exitCode?: number; message?: string }); - - expect((err as { exitCode?: number }).exitCode).toBe(3); - }); - - it('RATE_LIMITED during polling → non-auth, batch exits 1', async () => { - const creds = makeCreds(); - const passRun = makeTerminalRun('run_pass_a2', 'passed'); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_rl', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_pass_a2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_rl')) return errorBody('RATE_LIMITED'); - if (url.includes('/runs/run_pass_a2')) return { body: passRun }; - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(1); - }); - - it('NOT_FOUND during run polling → non-auth, batch exits 1', async () => { - const creds = makeCreds(); - const passRun = makeTerminalRun('run_pass_a3', 'passed'); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_nf', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_pass_a3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_nf')) return errorBody('NOT_FOUND'); - if (url.includes('/runs/run_pass_a3')) return { body: passRun }; - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(1); - }); -}); - -// --------------------------------------------------------------------------- -// [fix-auth-escalation] batch auth failure escalates to exit 3 -// --------------------------------------------------------------------------- - -describe('[fix-auth-escalation] auth error in batch rerun polling escalates to exit 3', () => { - it('auth failure in batch poll → batch exits 3, not 1', async () => { - const creds = makeCreds(); - const passRun = makeTerminalRun('run_other', 'passed'); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_auth', runId: 'run_auth', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_other', runId: 'run_other', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_auth')) return errorBody('AUTH_REQUIRED'); - if (url.includes('/runs/run_other')) return { body: passRun }; - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_auth', 'test_other'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(3); - }); - - it('mixed batch: one pass, one auth failure → exits 3 (auth wins)', async () => { - const creds = makeCreds(); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_pass', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_auth2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - const passRun = makeTerminalRun('run_pass', 'passed'); - passRun.testId = 'test_1'; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_pass')) return { body: passRun }; - if (url.includes('/runs/run_auth2')) return errorBody('AUTH_REQUIRED'); - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(3); - expect((err as { message?: string }).message).toMatch(/auth error/i); - }); - - it('non-auth failure → exits 1 (no escalation)', async () => { - const creds = makeCreds(); - const failRun = makeTerminalRun('run_fail', 'failed'); - failRun.testId = 'test_1'; - const passRun = makeTerminalRun('run_pass_c3', 'passed'); - passRun.testId = 'test_2'; - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_pass_c3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, ], deferred: [], conflicts: [], closure: { byProject: [] }, }; - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_fail')) return { body: failRun }; - if (url.includes('/runs/run_pass_c3')) return { body: passRun }; + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } return errorBody('NOT_FOUND'); }); + const stdoutLines: string[] = []; const err = await runTestRerun( { testIds: ['test_1', 'test_2'], all: false, wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(1); - }); -}); - -// --------------------------------------------------------------------------- -// [fix-D4] initial chunk idempotency key bounded to ≤256 chars -// --------------------------------------------------------------------------- - -describe('[fix-D4] initial chunk dispatch idempotency key bounded to 256 chars', () => { - it('short key with multiple chunks passes through unchanged', async () => { - const creds = makeCreds(); - const receivedKeys: string[] = []; - - // 51 test IDs forces 2 chunks (MAX_BATCH_RERUN_IDS = 50) - const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); - const batchResp: BatchRerunResponse = { - accepted: testIds.slice(0, 50).map(id => ({ - testId: id, - runId: `run_${id}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - })), - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - const batchResp2: BatchRerunResponse = { - accepted: [ - { - testId: testIds[50]!, - runId: `run_${testIds[50]}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - let callCount = 0; - - const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/batch/rerun')) { - const h = new Headers(init.headers ?? {}); - const key = h.get('idempotency-key') ?? ''; - receivedKeys.push(key); - callCount++; - return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; - } - return errorBody('NOT_FOUND'); - }); - - await runTestRerun( - { - testIds, - all: false, - wait: false, - timeoutSeconds: 600, + timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false, @@ -5149,154 +4861,23 @@ describe('[fix-D4] initial chunk dispatch idempotency key bounded to 256 chars', dryRun: false, debug: false, verbose: false, - idempotencyKey: 'short-key', }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - - expect(receivedKeys).toHaveLength(2); - expect(receivedKeys[0]).toBe('short-key:chunk0'); - expect(receivedKeys[1]).toBe('short-key:chunk1'); - expect(receivedKeys[0]!.length).toBeLessThanOrEqual(256); - expect(receivedKeys[1]!.length).toBeLessThanOrEqual(256); - }); - - it('249-char key + :chunk0 suffix would exceed 256 → key truncated to keep total ≤256', async () => { - const creds = makeCreds(); - const receivedKeys: string[] = []; - - // key is 249 chars; `:chunk0` is 7 chars → 256 total (edge case, fits exactly) - const longKey = 'k'.repeat(249); - const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); - const batchResp: BatchRerunResponse = { - accepted: testIds.slice(0, 50).map(id => ({ - testId: id, - runId: `run_${id}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - })), - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - const batchResp2: BatchRerunResponse = { - accepted: [ - { - testId: testIds[50]!, - runId: `run_${testIds[50]}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - let callCount = 0; - - const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/batch/rerun')) { - const h = new Headers(init.headers ?? {}); - receivedKeys.push(h.get('idempotency-key') ?? ''); - callCount++; - return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; - } - return errorBody('NOT_FOUND'); - }); - - await runTestRerun( { - testIds, - all: false, - wait: false, - timeoutSeconds: 600, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 10, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - idempotencyKey: longKey, + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - - expect(receivedKeys).toHaveLength(2); - for (const key of receivedKeys) { - expect(key.length).toBeLessThanOrEqual(256); - } - // suffix must be preserved - expect(receivedKeys[0]).toMatch(/:chunk0$/); - expect(receivedKeys[1]).toMatch(/:chunk1$/); - }); - - it('256-char key + :chunk0 suffix → base truncated so total is exactly 256', async () => { - const creds = makeCreds(); - const receivedKeys: string[] = []; + ).catch(e => e); - // Max-length user key: 256 chars. `:chunk0` = 7 chars → need to truncate base to 249. - const maxKey = 'x'.repeat(256); - const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); - const batchResp: BatchRerunResponse = { - accepted: testIds.slice(0, 50).map(id => ({ - testId: id, - runId: `run_${id}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - })), - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - const batchResp2: BatchRerunResponse = { - accepted: [ - { - testId: testIds[50]!, - runId: `run_${testIds[50]}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; }; - let callCount = 0; - - const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/batch/rerun')) { - const h = new Headers(init.headers ?? {}); - receivedKeys.push(h.get('idempotency-key') ?? ''); - callCount++; - return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; - } - return errorBody('NOT_FOUND'); - }); - - await runTestRerun( - { - testIds, - all: false, - wait: false, - timeoutSeconds: 600, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 10, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - idempotencyKey: maxKey, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - - expect(receivedKeys).toHaveLength(2); - for (const key of receivedKeys) { - expect(key.length).toBeLessThanOrEqual(256); - } - expect(receivedKeys[0]).toMatch(/:chunk0$/); - expect(receivedKeys[1]).toMatch(/:chunk1$/); + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 2f2ea9d..c0ff083 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -5,7 +5,7 @@ * sleep injection is wired through `TestDeps.sleep` to avoid real delays. */ -import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; @@ -3693,60 +3693,33 @@ describe('dashboardUrl on run completion', () => { }); }); -describe('runTestRunAll — JUnit report export', () => { - const BATCH_FRESH_RESP: BatchRunFreshResponse = { - accepted: [ - { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, - { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, - ], - conflicts: [], - deferred: [], - skippedFrontend: [], - skippedIntegration: [], - }; - - function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { - return { - runId, - testId, - projectId: 'project_be', - userId: 'user_1', - status: status as RunResponse['status'], - source: 'cli', - createdAt: '2026-06-09T10:00:00.000Z', - startedAt: '2026-06-09T10:00:01.000Z', - finishedAt: '2026-06-09T10:00:30.000Z', - codeVersion: 'v1', - targetUrl: 'https://api.example.com', - createdFrom: 'cli', - failedStepIndex: null, - failureKind: null, - error: null, - videoUrl: null, - stepSummary: { - total: 3, - completed: 3, - passedCount: status === 'passed' ? 3 : 0, - failedCount: 0, - }, - }; - } +// --------------------------------------------------------------------------- +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty +// --------------------------------------------------------------------------- - it('--wait --report junit writes XML after polling', async () => { +describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { const { credentialsPath } = makeCreds(); - const dir = mkdtempSync(join(tmpdir(), 'junit-run-all-')); - const reportPath = join(dir, 'results.xml'); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; const fetchImpl = makeFetch((url, init) => { - if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; - const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; - if (runId === 'run_fresh_01') - return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; - if (runId === 'run_fresh_02') - return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'passed') }; + if ((init.method ?? 'GET') === 'POST') return { body: batchResp }; + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_all'); + } return errorBody('NOT_FOUND'); }); + const stdoutLines: string[] = []; - await runTestRunAll( + const err = await runTestRunAll( { profile: 'default', output: 'json', @@ -3755,158 +3728,23 @@ describe('runTestRunAll — JUnit report export', () => { wait: true, timeoutSeconds: 60, maxConcurrency: 5, - report: 'junit', - reportFile: reportPath, }, { credentialsPath, fetchImpl, - stdout: () => undefined, + stdout: line => stdoutLines.push(line), stderr: () => undefined, sleep: instantSleep, }, - ); - - const xml = readFileSync(reportPath, 'utf8'); - expect(xml).toContain(' { - const { credentialsPath } = makeCreds(); - const dir = mkdtempSync(join(tmpdir(), 'junit-run-fail-')); - const reportPath = join(dir, 'results.xml'); - const fetchImpl = makeFetch((url, init) => { - if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; - const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; - if (runId === 'run_fresh_01') - return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; - if (runId === 'run_fresh_02') - return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'failed') }; - return errorBody('NOT_FOUND'); - }); - - await expect( - runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_be', - wait: true, - timeoutSeconds: 60, - maxConcurrency: 5, - report: 'junit', - reportFile: reportPath, - }, - { - credentialsPath, - fetchImpl, - stdout: () => undefined, - stderr: () => undefined, - sleep: instantSleep, - }, - ), - ).rejects.toMatchObject({ exitCode: 1 }); - - const xml = readFileSync(reportPath, 'utf8'); - expect(xml).toContain('failures="1"'); - expect(xml).toContain('name="test_be_02"'); - }); - - it('rejects --report without --wait', async () => { - await expect( - runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_be', - wait: false, - timeoutSeconds: 60, - maxConcurrency: 5, - report: 'junit', - reportFile: './results.xml', - }, - {}, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); - - it('rejects --report-suite-name without --report', async () => { - await expect( - runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_be', - wait: true, - timeoutSeconds: 60, - maxConcurrency: 5, - reportSuiteName: 'orphan-suite', - }, - {}, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); - - it('--dry-run --report junit writes canned sample XML', async () => { - const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-')); - const reportPath = join(dir, 'results.xml'); - - await runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - dryRun: true, - projectId: 'project_be', - wait: true, - timeoutSeconds: 60, - maxConcurrency: 5, - report: 'junit', - reportFile: reportPath, - }, - { - stdout: () => undefined, - stderr: () => undefined, - }, - ); - - const xml = readFileSync(reportPath, 'utf8'); - expect(xml).toContain('name="test_fresh_wave_01"'); - expect(xml).toContain('failures="1"'); - }); - - it('--dry-run --report junit --report-suite-name overrides canned suite name', async () => { - const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-suite-')); - const reportPath = join(dir, 'results.xml'); - - await runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - dryRun: true, - projectId: 'project_be', - wait: true, - timeoutSeconds: 60, - maxConcurrency: 5, - report: 'junit', - reportFile: reportPath, - reportSuiteName: 'ci-checkout-suite', - }, - { - stdout: () => undefined, - stderr: () => undefined, - }, - ); + ).catch(e => e); - const xml = readFileSync(reportPath, 'utf8'); - expect(xml).toContain('; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_fresh_01', 'run_fresh_02']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); diff --git a/src/commands/test.ts b/src/commands/test.ts index f11e688..9982651 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5951,6 +5951,22 @@ export async function runTestRunAll( }, }; } + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7) so the fan-out completes and stdout carries every runId. + // Without this, RequestTimeoutError rejects the fan-out before out.print(), + // leaving JSON consumers with empty stdout (mirrors create-batch --run). + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: err.message, + exitCode: err.exitCode, + }, + }; + } if (err instanceof ApiError) { // Preserve the real exit code + envelope (AUTH_INVALID=3, NOT_FOUND=4, // RATE_LIMITED=11, …) instead of flattening every member failure to 1 @@ -7160,6 +7176,22 @@ export async function runTestRerun( }, }; } + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7) so the fan-out completes and stdout carries every runId. + // Without this, RequestTimeoutError rejects the fan-out before out.print(), + // leaving JSON consumers with empty stdout (mirrors create-batch --run). + return { + testId: entry.testId, + runId: entry.runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: err.message, + exitCode: err.exitCode, + }, + }; + } if (err instanceof ApiError) { // Preserve the real exit code (AUTH_INVALID=3, RATE_LIMITED=11, …) so the // batch exit-code aggregator can escalate auth failures correctly. Mirroring From 8ac2ff0132e01c2c63e4a411ba4c0fbfc48a7a1d Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:12:12 -0400 Subject: [PATCH 057/117] =?UTF-8?q?feat(test):=20make=20"test=20wait"=20va?= =?UTF-8?q?riadic=20=E2=80=94=20attach=20to=20several=20runs=20in=20one=20?= =?UTF-8?q?invocation=20(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(test): make "test wait" variadic — attach to several runs in one invocation * fix(wait): honor the shared deadline for queued members and hint only resumable runs --- src/commands/test.test.ts | 155 +++++++++++ src/commands/test.ts | 247 +++++++++++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 12 +- 3 files changed, 404 insertions(+), 10 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index c78c7a8..0368e06 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -36,6 +36,7 @@ import { runPlanPut, runResult, runSteps, + runTestWaitMany, runUpdate, } from './test.js'; @@ -3042,6 +3043,160 @@ describe('runLint', () => { }); }); +describe('runTestWaitMany', () => { + const terminalRun = (runId: string, status: string) => ({ + runId, + testId: `test_of_${runId}`, + projectId: 'project_alice', + userId: 'u1', + status, + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }); + + it('polls every run, keeps input order, and exits 1 when one finished non-passed', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => ({ + body: url.includes('run_bad') + ? terminalRun('run_bad', 'failed') + : terminalRun('run_ok', 'passed'), + })); + const out: string[] = []; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_ok', 'run_bad'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + summary: { passed: number; failed: number }; + }; + expect(payload.results.map(row => row.runId)).toEqual(['run_ok', 'run_bad']); + expect(payload.summary).toMatchObject({ passed: 1, failed: 1 }); + }); + + it('a member whose poll errors is captured as error: and the others survive (exit 7)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('run_gone')) { + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'no such run', + nextAction: 'check the id', + requestId: 'req_x', + details: {}, + }, + }, + }; + } + return { body: terminalRun('run_ok', 'passed') }; + }); + const out: string[] = []; + const errs: string[] = []; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_ok', 'run_gone'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => errs.push(line), + }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + }; + // The failing member did not abort the pool: the passed verdict survived. + expect(payload.results[0]).toMatchObject({ runId: 'run_ok', status: 'passed' }); + expect(payload.results[1]!.status).toBe('error:NOT_FOUND'); + // The re-attach hint names the errored member (resumable) but NOT the + // already-terminal passed one. + const hint = errs.find(line => line.includes('Re-attach with:')); + expect(hint).toContain('run_gone'); + expect(hint).not.toContain('run_ok'); + }); + + it('members dequeued after the shared deadline are not granted extra poll time', async () => { + const { credentialsPath } = makeCreds(); + let fetches = 0; + const fetchImpl = makeFetch(() => { + fetches += 1; + return { body: terminalRun('run_any', 'passed') }; + }); + // timeoutSeconds 0: the shared deadline is already in the past when the + // pool starts, so every member must resolve to timeout WITHOUT polling + // (previously each dequeued member was granted a fresh 1s minimum). + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_a', 'run_b', 'run_c'], + timeoutSeconds: 0, + maxConcurrency: 1, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + expect(fetches).toBe(0); + }); + + it('an auth error escalates the exit code to 3', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 401, + body: { + error: { + code: 'AUTH_INVALID', + message: 'bad key', + nextAction: 'run setup', + requestId: 'req_y', + details: {}, + }, + }, + })); + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_a', 'run_b'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 3 }); + }); +}); + describe('runResult', () => { it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index 9982651..57d90f5 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5331,6 +5331,211 @@ export async function runTestRun( return finalRun; } +/** One row of the `test wait ` multi-run payload. */ +export interface CliMultiWaitResult { + runId: string; + /** Terminal run status, or 'timeout', or 'error:' when the poll failed. */ + status: string; + /** Test the run belongs to, when the poll observed it. */ + testId?: string; +} + +export interface RunTestWaitManyOptions extends CommonOptions { + runIds: string[]; + timeoutSeconds: number; + maxConcurrency: number; +} + +/** + * `test wait ` with two or more ids: attach to N already-dispatched + * runs in ONE invocation. This closes the loop the CLI itself opens: every + * batch/closure timeout prints one `testsprite test wait ` hint PER + * member, which previously meant N sequential blocking invocations. The runs + * are polled concurrently under a bounded pool with ONE shared deadline + * (`--timeout` bounds the whole invocation, not each member), each member's + * poll is total (a transient error on one run never discards the others), and + * the exit code is the worst status across members: auth errors escalate to + * exit 3, any timeout or poll error exits 7, any non-passed terminal exits 1. + * Distinct from a run journal (issue #80): no persistence, just N known ids. + */ +export async function runTestWaitMany( + opts: RunTestWaitManyOptions, + deps: TestDeps = {}, +): Promise<{ results: CliMultiWaitResult[]; summary: Record }> { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const results: CliMultiWaitResult[] = opts.runIds.map(runId => ({ + runId, + status: 'passed', + })); + const payload = { + results, + summary: { total: results.length, passed: results.length, failed: 0, timedOut: 0, errors: 0 }, + }; + out.print(payload, () => results.map(r => `${r.runId} ${r.status}`).join('\n')); + return payload; + } + + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + + // One shared deadline across every member (the whole point of the shared + // pool: `--timeout 600` means the invocation ends within ~600s, not + // 600s x ceil(N/concurrency)). + const deadlineMs = Date.now() + opts.timeoutSeconds * 1000; + + type WaitOutcome = + | { kind: 'result'; run: RunResponse } + | { kind: 'timeout' } + | { kind: 'error'; code: string; exitCode: number }; + + const pollOne = async (runId: string): Promise => { + // A member dequeued AFTER the shared deadline has passed must not be + // granted a fresh minimum poll window (with --max-concurrency 1 that + // would extend the invocation by ~1s per queued run past --timeout). + const remainingSeconds = Math.ceil((deadlineMs - Date.now()) / 1000); + if (remainingSeconds <= 0) return { kind: 'timeout' }; + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: run => run.testId, + resolveNotBefore: run => run.createdAt, + onResolved: () => undefined, + }); + try { + const run = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: remainingSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + ticker.update(`Run ${run.runId} — ${run.status} (elapsed=${elapsed}s)`); + }, + resolveAlternate, + }); + return { kind: 'result', run }; + } catch (err) { + if (err instanceof TimeoutError) return { kind: 'timeout' }; + if (err instanceof RequestTimeoutError) throw err; + if (err instanceof ApiError) return { kind: 'error', code: err.code, exitCode: err.exitCode }; + return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; + } + }; + + const outcomes = new Map(); + let inFlight = 0; + let nextIdx = 0; + try { + await new Promise((resolve, reject) => { + const startNext = (): void => { + while (inFlight < opts.maxConcurrency && nextIdx < opts.runIds.length) { + const runId = opts.runIds[nextIdx++]!; + inFlight++; + pollOne(runId) + .then(outcome => { + outcomes.set(runId, outcome); + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= opts.runIds.length) resolve(); + }) + // pollOne is total except for RequestTimeoutError (handled below). + .catch(reject); + } + }; + startNext(); + if (opts.runIds.length === 0) resolve(); + }); + } catch (fanOutErr) { + if (fanOutErr instanceof RequestTimeoutError) { + // Same contract as the batch pollers: leave stdout parseable before + // exiting 7. Members that already settled keep their real status; only + // the still-unfinished ids are marked running and named in the hint + // (re-attaching to an already-terminal run would be a wasted command). + ticker.finalize('Multi-run wait — request timed out'); + const partial = { + results: opts.runIds.map((runId): CliMultiWaitResult => { + const outcome = outcomes.get(runId); + if (outcome === undefined) return { runId, status: 'running' }; + if (outcome.kind === 'timeout') return { runId, status: 'timeout' }; + if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; + return { runId, status: outcome.run.status, testId: outcome.run.testId }; + }), + summary: { total: opts.runIds.length }, + }; + out.print(partial, () => partial.results.map(r => `${r.runId} ${r.status}`).join('\n')); + const unfinished = partial.results + .filter(r => r.status === 'running' || r.status === 'timeout') + .map(r => r.runId); + if (unfinished.length > 0) { + stderrFn(`Re-attach with: testsprite test wait ${unfinished.join(' ')}`); + } + } + throw fanOutErr; + } + ticker.finalize(); + + const results: CliMultiWaitResult[] = opts.runIds.map(runId => { + const outcome = outcomes.get(runId); + if (outcome === undefined || outcome.kind === 'timeout') return { runId, status: 'timeout' }; + if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; + return { runId, status: outcome.run.status, testId: outcome.run.testId }; + }); + const passed = results.filter(r => r.status === 'passed').length; + const timedOut = results.filter(r => r.status === 'timeout').length; + const errors = results.filter(r => r.status.startsWith('error:')).length; + const failed = results.length - passed - timedOut - errors; + const payload = { + results, + summary: { total: results.length, passed, failed, timedOut, errors }, + }; + out.print(payload, () => + [ + ...results.map(r => `${r.runId} ${r.status}`), + '', + `${passed}/${results.length} passed, ${failed} failed/blocked, ${timedOut} timed out, ${errors} poll errors`, + ].join('\n'), + ); + + // Every member that did not reach a terminal verdict is re-attachable: + // timeouts (still running server-side) and poll errors (e.g. a transient + // transport failure) both belong in the hint; terminal runs do not. + const unfinishedIds = results + .filter(r => r.status === 'timeout' || r.status.startsWith('error:')) + .map(r => r.runId); + if (unfinishedIds.length > 0) { + stderrFn(`Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}`); + } + + // Worst-status exit: auth escalates (a rejected key fails every member the + // same way), then timeout/poll-error (7, resumable), then plain failure (1). + const authError = [...outcomes.values()].find( + o => + o.kind === 'error' && + (o.code === 'AUTH_REQUIRED' || o.code === 'AUTH_INVALID' || o.code === 'AUTH_FORBIDDEN'), + ); + if (authError !== undefined && authError.kind === 'error') { + throw new CLIError( + `Multi-run wait: authentication failed (${authError.code})`, + authError.exitCode, + ); + } + if (timedOut > 0 || errors > 0) { + throw new CLIError( + `Multi-run wait: ${timedOut} timed out, ${errors} poll error(s) out of ${results.length} runs`, + 7, + ); + } + if (failed > 0) { + throw new CLIError(`Multi-run wait: ${failed} run(s) finished non-passed`, 1); + } + return payload; +} + /** * `test wait ` — M3.3 piece-3. * @@ -8172,26 +8377,53 @@ export function createTestCommand(deps: TestDeps = {}): Command { }); test - .command('wait ') + .command('wait ') .description( - 'Wait for a run to reach a terminal status.\n' + + 'Wait for one or more runs to reach a terminal status.\n' + + '\nWith several run-ids the runs are polled concurrently under one shared\n' + + '--timeout and a {results, summary} envelope is printed (worst status wins\n' + + 'the exit code), so every re-attach hint the CLI prints can be pasted as\n' + + 'ONE command.\n' + '\nExit codes:\n' + ' 0 passed\n' + ' 1 failed / blocked / cancelled\n' + ' 3 auth error\n' + - ' 4 run not found\n' + - ' 7 timeout — resume with: testsprite test wait \n' + + ' 4 run not found (single run-id; with several ids a per-member poll error\n' + + ' is recorded as error: in its row and folded into exit 7)\n' + + ' 7 timeout or per-member poll error — resume with: testsprite test wait \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', ) .option('--timeout ', `max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`) + .option( + '--max-concurrency ', + 'with several run-ids, max concurrent polls (1-100, default: 10)', + ) .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (runId: string, cmdOpts: WaitFlagOpts, command: Command) => { - await runTestWait( + .action(async (runIds: string[], cmdOpts: WaitFlagOpts, command: Command) => { + // One id keeps the historical single-run path byte-identical (same + // output shape, same exit codes); two or more fan out. + if (runIds.length === 1) { + await runTestWait( + { + ...resolveCommonOptions(command), + runId: runIds[0]!, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + return; + } + const maxConcurrency = parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? 10; + if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 100) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + await runTestWaitMany( { ...resolveCommonOptions(command), - runId, + runIds, timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + maxConcurrency, }, deps, ); @@ -8577,6 +8809,7 @@ interface RunFlagOpts { interface WaitFlagOpts { timeout?: string; + maxConcurrency?: string; } interface RerunFlagOpts { diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 9fb6b2d..a16d0c4 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -251,14 +251,20 @@ Commands: 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get - wait [options] Wait for a run to reach a terminal status. + wait [options] Wait for one or more runs to reach a terminal status. + + With several run-ids the runs are polled concurrently under one shared + --timeout and a {results, summary} envelope is printed (worst status wins + the exit code), so every re-attach hint the CLI prints can be pasted as + ONE command. Exit codes: 0 passed 1 failed / blocked / cancelled 3 auth error - 4 run not found - 7 timeout — resume with: testsprite test wait + 4 run not found (single run-id; with several ids a per-member poll error + is recorded as error: in its row and folded into exit 7) + 7 timeout or per-member poll error — resume with: testsprite test wait 10 transport/network failure (UNAVAILABLE) — retry the command On failure/blocked/cancelled, run: testsprite test artifact get From e819e4543e5993f7548965b15cd9a03c1ee87b95 Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Mon, 6 Jul 2026 21:12:45 +0100 Subject: [PATCH 058/117] feat(agent): add GitHub Copilot as an install target (#194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(agent): add GitHub Copilot as an install target Adds `copilot` to the agent-install targets. GitHub Copilot reads path-specific custom instructions from `.github/instructions/*.instructions.md` (VS Code / Visual Studio / Copilot Chat), with YAML frontmatter carrying an `applyTo` glob. The skill installs to `.github/instructions/testsprite-verify.instructions.md` (and the onboard skill alongside) with `applyTo: '**'` so the guidance attaches to every request in the repo. Because `applyTo: '**'` is always-on (Copilot has no on-demand 'model decides' mode like Cursor/Windsurf), the target renders the COMPACT body — the same reasoning behind windsurf's compact render — keeping the always-injected context small (~6 KB vs the ~23 KB full body). Slots into the existing TARGETS machinery, so `agent list`, `setup --agent`, and skill-nudge install-detection pick it up automatically. Updates the AgentTarget union, pathFor, TARGETS, help/docs, unit + e2e matrix guards, and the help snapshot. Fixes #193 * fix(agent): include Kiro in the agent command description The top-level `agent` command description listed the other targets but omitted Kiro (a pre-existing gap), leaving it inconsistent with the `--target` help text and the docs. Align the description with the `--target` list so every supported target is named. --- DOCUMENTATION.md | 7 +-- README.md | 44 ++++++++-------- src/commands/agent.test.ts | 15 +++--- src/commands/agent.ts | 4 +- src/lib/agent-targets.test.ts | 51 +++++++++++++++++-- src/lib/agent-targets.ts | 30 ++++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 13 ++--- test/e2e/agent-install.e2e.test.ts | 11 +++- test/e2e/setup.e2e.test.ts | 1 + 9 files changed, 131 insertions(+), 45 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index fb62cae..6b798ef 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -115,14 +115,15 @@ testsprite agent install cline # .clinerules/testsprite-verify.md testsprite agent install windsurf # .windsurf/rules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md -testsprite agent list # list all 7 targets with status + mode + path +testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md +testsprite agent list # list all 8 targets with status + mode + path ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental). +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. ## Command reference diff --git a/README.md b/README.md index 79f8aea..404f237 100644 --- a/README.md +++ b/README.md @@ -89,28 +89,28 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test wait` | Block on a `runId` until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 7da57cf..4f5fdd3 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -769,13 +769,14 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 7 targets × 2 default skills = 14 rows - expect(json).toHaveLength(14); + // 8 targets × 2 default skills = 16 rows + expect(json).toHaveLength(16); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); expect(targets).toContain('windsurf'); + expect(targets).toContain('copilot'); expect(targets).toContain('antigravity'); expect(targets).toContain('kiro'); expect(targets).toContain('codex'); @@ -949,11 +950,11 @@ describe('runInstall — all own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all six own-file targets +// Dry-run for all seven own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the six own-file targets (default 2 skills = 12 would-write lines)', async () => { + it('writes nothing for any of the seven own-file targets (default 2 skills = 14 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -963,7 +964,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf', 'copilot'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -973,9 +974,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 6 targets × 2 default skills = 12 would-write lines + // 7 targets × 2 default skills = 14 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(12); + expect(wouldWriteLines.length).toBe(14); }); }); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 0e80a91..0a40c4b 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -1061,7 +1061,7 @@ function collect(v: string, prev: string[]): string[] { export function createAgentCommand(deps: AgentDeps = {}): Command { const agent = new Command('agent').description( - 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Windsurf, Antigravity, Codex)', + 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Kiro, Windsurf, Copilot, Codex)', ); agent @@ -1071,7 +1071,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, copilot, codex (comma-separated or repeated)', collect, [], ) diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 0417166..51bf9b5 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -80,19 +80,29 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all seven required keys', () => { + it('has all eight required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro', 'windsurf']); + expect(keys).toEqual([ + 'antigravity', + 'claude', + 'cline', + 'codex', + 'copilot', + 'cursor', + 'kiro', + 'windsurf', + ]); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, windsurf, antigravity, kiro, and codex are experimental', () => { + it('cursor, cline, windsurf, copilot, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); expect(TARGETS.windsurf.status).toBe('experimental'); + expect(TARGETS.copilot.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); @@ -112,6 +122,7 @@ describe('TARGETS', () => { expect(TARGETS.cline.mode).toBe('own-file'); expect(TARGETS.kiro.mode).toBe('own-file'); expect(TARGETS.windsurf.mode).toBe('own-file'); + expect(TARGETS.copilot.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -341,11 +352,45 @@ describe('windsurf renders within the rules-file budget', () => { }); }); +describe('renderForTarget("copilot")', () => { + const result = renderForTarget('copilot', 'testsprite-verify', STUB_BODY); + + it('returns the .github/instructions path', () => { + expect(result.path).toBe('.github/instructions/testsprite-verify.instructions.md'); + }); + + it('uses the Copilot frontmatter (applyTo + description)', () => { + expect(result.content.startsWith('---\n')).toBe(true); + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + expect(result.content).toContain("applyTo: '**'"); + }); + + it('does NOT carry the Claude/Cursor/Windsurf frontmatter keys', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); // claude key + expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key + expect(fm).not.toContain('trigger:'); // windsurf Cascade key + }); + + it('renders the compact verify body (applyTo:** is always-on, so keep it small)', () => { + // Uses the REAL bodies (no stub): copilot always-injects, so like windsurf it + // ships the trimmed verify body while keeping the load-bearing command. + const copilot = renderForTarget('copilot', 'testsprite-verify'); + const claude = renderForTarget('claude', 'testsprite-verify'); + expect(copilot.content.length).toBeLessThan(claude.content.length); + expect(copilot.content).not.toContain('The verification loop that flies'); + expect(copilot.content).toContain('testsprite test run'); + }); +}); + // --------------------------------------------------------------------------- // Content integrity — load-bearing command strings must survive any body trim // --------------------------------------------------------------------------- describe('content integrity — own-file targets', () => { + // Full-body own-file targets. Compact-body targets (windsurf, copilot) are + // excluded — they render the trimmed verify body; see their dedicated tests. const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity' | 'kiro'> = [ 'claude', 'cursor', diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index d9a85d5..7e6f94d 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -9,7 +9,8 @@ export type AgentTarget = | 'antigravity' | 'codex' | 'kiro' - | 'windsurf'; + | 'windsurf' + | 'copilot'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -149,6 +150,19 @@ function wrapWindsurf(_name: string, description: string, body: string): string return `---\ntrigger: model_decision\ndescription: ${description}\n---\n\n${body}\n`; } +/** + * GitHub Copilot reads path-specific custom instructions from + * `.github/instructions/*.instructions.md` (VS Code / Visual Studio / GitHub + * Copilot Chat). Each file carries YAML frontmatter with `applyTo` — a glob that + * scopes when the instructions attach. `applyTo: '**'` attaches the guidance to + * every request in the repo, which is what a persistent verification skill wants + * (there is no on-demand "model decides" mode for Copilot instruction files, so + * always-apply is the correct idiom). `description` is surfaced in Copilot's UI. + */ +function wrapCopilot(_name: string, description: string, body: string): string { + return `---\ndescription: ${description}\napplyTo: '**'\n---\n\n${body}\n`; +} + // --------------------------------------------------------------------------- // Landing paths // --------------------------------------------------------------------------- @@ -173,6 +187,8 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.kiro/skills/${skill}/SKILL.md`; case 'windsurf': return `.windsurf/rules/${skill}.md`; + case 'copilot': + return `.github/instructions/${skill}.instructions.md`; case 'codex': return 'AGENTS.md'; } @@ -220,6 +236,18 @@ export const TARGETS: Record = { compactBody: true, wrap: wrapWindsurf, }, + copilot: { + status: 'experimental', + path: pathFor('copilot', SKILL_NAME), + mode: 'own-file', + // GitHub Copilot path-specific instructions: frontmatter carries `applyTo`. + // `applyTo: '**'` means the file is ALWAYS injected into Copilot requests + // (there is no on-demand "model decides" mode like Cursor/Windsurf), so + // render the compact body to keep the always-on context cost small — the + // same reasoning that drives windsurf's compact render. + compactBody: true, + wrap: wrapCopilot, + }, /** * codex target — managed-section mode. * diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index a16d0c4..231d531 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -4,7 +4,7 @@ exports[`--help snapshots > agent 1`] = ` "Usage: testsprite agent [options] [command] Install TestSprite guidance into coding-agent config (Claude Code, Cursor, -Cline, Windsurf, Antigravity, Codex) +Cline, Antigravity, Kiro, Windsurf, Copilot, Codex) Options: -h, --help display help for command @@ -29,7 +29,8 @@ into a project for a coding agent Options: --target Agent target(s): claude, cursor, cline, antigravity, kiro, - windsurf, codex (comma-separated or repeated) (default: []) + windsurf, copilot, codex (comma-separated or repeated) + (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -115,8 +116,8 @@ Options: --from-env Read TESTSPRITE_API_KEY from the environment instead of prompting (default: false) --agent Coding-agent target to install: claude, antigravity, - cursor, cline, kiro, windsurf, codex (default: claude) - (default: "claude") + cursor, cline, kiro, windsurf, copilot, codex (default: + claude) (default: "claude") --no-agent Skip the agent skill install (configure credentials only) --force Overwrite an existing skill file (a .bak backup is kept) --dir Project root for the skill install (default: current @@ -656,8 +657,8 @@ Commands: project Manage TestSprite projects test Inspect TestSprite tests agent Install TestSprite guidance into coding-agent - config (Claude Code, Cursor, Cline, Windsurf, - Antigravity, Codex) + config (Claude Code, Cursor, Cline, Antigravity, + Kiro, Windsurf, Copilot, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) help [command] display help for command diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 094aec2..4fb4584 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -174,6 +174,11 @@ describe('content integrity', () => { expect(content.startsWith('---'), `windsurf: should start with ---`).toBe(true); expect(content).toContain('trigger: model_decision'); expect(content).toContain('description:'); + } else if (target === 'copilot') { + // GitHub Copilot instructions frontmatter: applyTo glob + description + expect(content.startsWith('---'), `copilot: should start with ---`).toBe(true); + expect(content).toContain("applyTo: '**'"); + expect(content).toContain('description:'); } // (b) branding — the renamed H1 must be present in every body variant @@ -211,6 +216,9 @@ describe('content integrity', () => { expect(content.startsWith('---'), `windsurf/onboard: should start with ---`).toBe(true); expect(content).toContain('trigger: model_decision'); expect(content).toContain('description:'); + } else if (target === 'copilot') { + expect(content.startsWith('---'), `copilot/onboard: should start with ---`).toBe(true); + expect(content).toContain("applyTo: '**'"); } // Load-bearing onboard string: the skill body must reference setup @@ -803,7 +811,7 @@ describe('agent list', () => { }>; expect(Array.isArray(parsed)).toBe(true); - // Expected: 7 targets × 2 skills = 14 rows + // Expected: 8 targets × 2 skills = 16 rows const expectedCount = Object.keys(TARGETS).length * DEFAULT_SKILLS.length; expect(parsed.length).toBe(expectedCount); @@ -839,6 +847,7 @@ describe('matrix coverage guard', () => { 'cline', 'kiro', 'windsurf', + 'copilot', 'codex', ]); }); diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index 3b35751..5a07434 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -229,6 +229,7 @@ describe('matrix coverage guard', () => { 'cline', 'kiro', 'windsurf', + 'copilot', 'codex', ]); }); From edc31c8e0e82599a9b95d15ba761eede61683517 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:13:18 -0400 Subject: [PATCH 059/117] feat(cli): add "testsprite doctor" environment diagnostic (#183) * feat(cli): add "testsprite doctor" environment diagnostic One-shot preflight: checks CLI version, Node runtime, profile, API endpoint, credentials, live connectivity (GET /me), and verify-skill install. Prints an OK/WARN/FAIL report and exits non-zero when any check fails, so it gates a CI step or agent preflight. Reuses the real resolution helpers; the API key is never printed. Fixes #73 * fix(doctor): address review findings - Node check reuses the CLI runtime guard (shouldRejectNodeVersion) instead of a hardcoded major floor, so the verdict matches what the entrypoint enforces at startup; the precise 20.19+/22.13+/24+ engines are enforced by npm engine-strict. - --request-timeout raises a validation error on malformed input instead of silently defaulting, matching the other commands. - The --output json test asserts the API key never appears in the JSON path (distinct from the text renderer already covered). --- src/commands/doctor.test.ts | 229 ++++++++++++++ src/commands/doctor.ts | 285 ++++++++++++++++++ src/index.ts | 2 + test/__snapshots__/help.snapshot.test.ts.snap | 2 + 4 files changed, 518 insertions(+) create mode 100644 src/commands/doctor.test.ts create mode 100644 src/commands/doctor.ts diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts new file mode 100644 index 0000000..230398a --- /dev/null +++ b/src/commands/doctor.test.ts @@ -0,0 +1,229 @@ +/** + * Unit tests for `testsprite doctor`. + * + * The command reuses the real resolution helpers (loadConfig, makeHttpClient, + * isVerifySkillInstalled), so these tests inject env/credentials/fetch/fs and + * assert on the rendered report + the exit-on-failure contract. + */ + +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CLIError } from '../lib/errors.js'; +import { writeProfile } from '../lib/credentials.js'; +import type { DoctorDeps, DoctorReport } from './doctor.js'; +import { createDoctorCommand, runDoctor } from './doctor.js'; + +interface CapturedOutput { + stdout: string[]; + stderr: string[]; +} + +function makeCapture(): { capture: CapturedOutput; deps: Pick } { + const capture: CapturedOutput = { stdout: [], stderr: [] }; + return { + capture, + deps: { + stdout: line => capture.stdout.push(line), + stderr: line => capture.stderr.push(line), + }, + }; +} + +function makeFetch(body: unknown, status = 200): DoctorDeps['fetchImpl'] { + return vi.fn( + async () => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as DoctorDeps['fetchImpl']; +} + +const OK_ME = { userId: 'u-doc', keyId: 'k-doc' }; + +/** Base deps shared by the healthy-path tests: node OK, skill installed, empty env. */ +function healthyDeps(credentialsPath: string, extra: Partial = {}): DoctorDeps { + return { + env: {}, + credentialsPath, + cwd: '/project', + nodeVersion: '22.9.0', + existsSync: () => true, // skill landing file present + fetchImpl: makeFetch(OK_ME), + ...extra, + }; +} + +let credentialsPath: string; + +beforeEach(() => { + credentialsPath = join(mkdtempSync(join(tmpdir(), 'testsprite-doctor-')), 'credentials'); +}); + +describe('runDoctor — healthy environment', () => { + it('returns an all-passing report and does not throw', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + expect(report.failures).toBe(0); + expect(report.warnings).toBe(0); + const out = capture.stdout.join('\n'); + expect(out).toContain('[OK]'); + expect(out).toContain('All checks passed.'); + expect(out).toContain('reached GET /me'); + }); + + it('never prints the API key anywhere in the report', async () => { + writeProfile('default', { apiKey: 'sk-super-secret-value' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + const all = capture.stdout.join('\n') + capture.stderr.join('\n'); + expect(all).not.toContain('sk-super-secret-value'); + }); + + it('emits a machine-readable report under --output json without leaking the API key', async () => { + writeProfile('default', { apiKey: 'sk-json-secret-value' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runDoctor( + { profile: 'default', output: 'json', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + const raw = capture.stdout.join(''); + // Security: the JSON serialization path is distinct from the text renderer, + // so assert the key never leaks here either. + expect(raw).not.toContain('sk-json-secret-value'); + const parsed = JSON.parse(raw) as DoctorReport; + expect(parsed.failures).toBe(0); + expect(Array.isArray(parsed.checks)).toBe(true); + expect( + parsed.checks.some(check => check.name === 'Connectivity' && check.status === 'ok'), + ).toBe(true); + }); +}); + +describe('runDoctor — failing checks exit non-zero', () => { + it('missing API key fails Credentials and throws CLIError (exit 1)', async () => { + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, // no profile written => no key + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + expect(rejection).toMatchObject({ exitCode: 1 }); + const out = capture.stdout.join('\n'); + expect(out).toContain('[FAIL]'); + expect(out).toContain('Credentials'); + }); + + it('invalid endpoint URL fails the API endpoint check', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false, endpointUrl: 'not-a-url' }, + { ...healthyDeps(credentialsPath), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('API endpoint'); + expect(out).toContain('not a valid'); + }); + + it('rejected API key surfaces as a Connectivity failure', async () => { + writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const authError = { + error: { code: 'AUTH_INVALID', message: 'Bad key.', requestId: 'req_x', details: {} }, + }; + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(authError, 401) }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('Connectivity'); + expect(out).toContain('API key rejected (AUTH_INVALID)'); + }); + + it('a non-auth /me error is reported as a Connectivity failure with its code', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const notFound = { + error: { code: 'NOT_FOUND', message: 'nope', requestId: 'req_y', details: {} }, + }; + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(notFound, 404) }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + expect(capture.stdout.join('\n')).toContain('GET /me failed (NOT_FOUND)'); + }); + + it('an outdated Node runtime fails the Node.js check', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { nodeVersion: '18.0.0' }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('Node.js'); + expect(out).toContain('below the required Node 20'); + }); +}); + +describe('runDoctor — warnings do not fail', () => { + it('missing verify skill is a warning, not a failure', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { existsSync: () => false }), ...deps }, + ); + expect(report.failures).toBe(0); + expect(report.warnings).toBeGreaterThanOrEqual(1); + const out = capture.stdout.join('\n'); + expect(out).toContain('[WARN]'); + expect(out).toContain('Verify skill'); + }); + + it('--dry-run skips connectivity and never calls fetch, missing key is a warning', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('fetch must not be called under --dry-run'); + }) as unknown as DoctorDeps['fetchImpl']; + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + { + env: {}, + credentialsPath, + cwd: '/project', + nodeVersion: '22.9.0', + existsSync: () => true, + fetchImpl, + ...deps, + }, + ); + expect(report.failures).toBe(0); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(capture.stdout.join('\n')).toContain('skipped under --dry-run'); + }); +}); + +describe('createDoctorCommand wiring', () => { + it('exposes the doctor command name', () => { + expect(createDoctorCommand().name()).toBe('doctor'); + }); + + it('--help describes the diagnostic', () => { + expect(createDoctorCommand().helpInformation()).toContain('Diagnose'); + }); +}); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts new file mode 100644 index 0000000..2fa0c57 --- /dev/null +++ b/src/commands/doctor.ts @@ -0,0 +1,285 @@ +/** + * `testsprite doctor` — one-shot environment diagnostic. + * + * Runs a fixed checklist (CLI version, Node.js runtime, active profile, API + * endpoint, credentials, live connectivity + key validity, and whether the + * verify skill is installed in the current project) and prints an OK/WARN/FAIL + * report. Exits non-zero when any check FAILS so it can gate a CI step or an + * agent preflight (`testsprite doctor && testsprite test run ...`). Warnings + * (e.g. skill not installed) do not fail the process. + * + * Every check is reused from the same helpers the real commands use, so the + * report reflects exactly what a subsequent command would resolve: `loadConfig` + * for profile/endpoint/key, `assertValidEndpointUrl` for the endpoint gate, + * `makeHttpClient` + `GET /me` for connectivity, and `isVerifySkillInstalled` + * for the skill check. + */ + +import { Command } from 'commander'; +import { + assertValidEndpointUrl, + makeHttpClient, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { loadConfig } from '../lib/config.js'; +import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; +import type { FetchImpl } from '../lib/http.js'; +import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; +import { VERSION } from '../version.js'; +import { MIN_SUPPORTED_NODE_MAJOR, shouldRejectNodeVersion } from '../version-guard.js'; + +export type DoctorStatus = 'ok' | 'warn' | 'fail'; + +export interface DoctorCheck { + /** Short, stable label (also the JSON key-ish name). */ + name: string; + status: DoctorStatus; + /** Human-readable one-line result. Never contains the API key. */ + detail: string; +} + +export interface DoctorReport { + checks: DoctorCheck[]; + failures: number; + warnings: number; +} + +/** Minimal projection of `GET /me` we read for the connectivity detail. */ +interface MeIdentity { + userId?: string; + keyId?: string; +} + +export interface DoctorDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + /** Project dir for the skill check. Defaults to `process.cwd()`. */ + cwd?: string; + /** Runtime version string (e.g. "22.9.0"). Defaults to `process.versions.node`. */ + nodeVersion?: string; + existsSync?: (p: string) => boolean; + readFileSync?: (p: string) => string; +} + +type CommonOptions = FactoryCommonOptions; + +export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const env = deps.env ?? process.env; + const cwd = deps.cwd ?? process.cwd(); + const nodeVersion = deps.nodeVersion ?? process.versions.node; + + const config = loadConfig({ + profile: opts.profile, + endpointUrl: opts.endpointUrl, + env, + credentialsPath: deps.credentialsPath, + }); + const endpointCheck = checkEndpoint(config.apiUrl); + const hasKey = Boolean(config.apiKey); + + const checks: DoctorCheck[] = [ + { name: 'CLI version', status: 'ok', detail: VERSION }, + checkNodeVersion(nodeVersion), + { name: 'Profile', status: 'ok', detail: config.profile }, + endpointCheck, + checkCredentials(hasKey, config.profile, opts.dryRun ?? false), + await checkConnectivity(opts, deps, { + hasKey, + endpointOk: endpointCheck.status === 'ok', + }), + checkSkill(cwd, deps), + ]; + + const failures = checks.filter(check => check.status === 'fail').length; + const warnings = checks.filter(check => check.status === 'warn').length; + const report: DoctorReport = { checks, failures, warnings }; + + out.print(report, () => renderDoctor(report)); + + if (failures > 0) { + // Non-zero exit so `testsprite doctor && ...` gates a CI step or an agent + // preflight. The full report already printed above; this line is the stderr + // summary index.ts renders before exiting 1. + throw new CLIError(`doctor: ${failures} check(s) failed, ${warnings} warning(s)`, 1); + } + return report; +} + +function checkNodeVersion(nodeVersion: string): DoctorCheck { + // Reuse the CLI's own runtime guard so the verdict matches exactly what the + // entrypoint enforces at startup, rather than a divergent hardcoded check. + // The precise engines floor (20.19+/22.13+/24+) is enforced by npm at install + // time via .npmrc engine-strict. sourceRef: src/version-guard.ts. + const rejected = shouldRejectNodeVersion(nodeVersion); + return { + name: 'Node.js', + status: rejected ? 'fail' : 'ok', + detail: rejected + ? `v${nodeVersion} is below the required Node ${MIN_SUPPORTED_NODE_MAJOR}; upgrade Node.js` + : `v${nodeVersion} (>=${MIN_SUPPORTED_NODE_MAJOR} required)`, + }; +} + +function checkEndpoint(apiUrl: string): DoctorCheck { + try { + assertValidEndpointUrl(apiUrl); + return { name: 'API endpoint', status: 'ok', detail: apiUrl }; + } catch { + return { + name: 'API endpoint', + status: 'fail', + detail: `"${apiUrl}" is not a valid http(s) URL`, + }; + } +} + +function checkCredentials(hasKey: boolean, profile: string, dryRun: boolean): DoctorCheck { + if (hasKey) { + // Never print any part of the key (security). Confirm presence only. + return { + name: 'Credentials', + status: 'ok', + detail: `API key configured (profile "${profile}")`, + }; + } + // Under --dry-run no key is expected, so a missing key is not a failure. + return { + name: 'Credentials', + status: dryRun ? 'warn' : 'fail', + detail: dryRun + ? 'no API key (not needed under --dry-run)' + : 'no API key found; run `testsprite setup` (or set TESTSPRITE_API_KEY)', + }; +} + +function checkSkill(cwd: string, deps: DoctorDeps): DoctorCheck { + const installed = isVerifySkillInstalled(cwd, { + existsSync: deps.existsSync, + readFileSync: deps.readFileSync, + }); + return { + name: 'Verify skill', + status: installed ? 'ok' : 'warn', + detail: installed + ? 'installed in this project' + : 'not installed here; run `testsprite setup` so your agent verifies its changes', + }; +} + +async function checkConnectivity( + opts: CommonOptions, + deps: DoctorDeps, + ctx: { hasKey: boolean; endpointOk: boolean }, +): Promise { + const name = 'Connectivity'; + if (opts.dryRun) return { name, status: 'warn', detail: 'skipped under --dry-run' }; + if (!ctx.hasKey) return { name, status: 'warn', detail: 'skipped; no API key to test with' }; + if (!ctx.endpointOk) return { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' }; + + try { + const client = makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); + const me = await client.get('/me'); + const who = me.userId ? ` (userId ${me.userId})` : ''; + return { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }; + } catch (error) { + if (error instanceof ApiError) { + if ( + error.code === 'AUTH_REQUIRED' || + error.code === 'AUTH_INVALID' || + error.code === 'AUTH_FORBIDDEN' + ) { + return { name, status: 'fail', detail: `API key rejected (${error.code})` }; + } + return { name, status: 'fail', detail: `GET /me failed (${error.code})` }; + } + return { + name, + status: 'fail', + detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, + }; + } +} + +const STATUS_LABEL: Record = { + ok: '[OK] ', + warn: '[WARN]', + fail: '[FAIL]', +}; + +function renderDoctor(report: DoctorReport): string { + const nameWidth = Math.max(...report.checks.map(check => check.name.length)); + const lines: string[] = ['TestSprite doctor', '']; + for (const check of report.checks) { + lines.push(` ${STATUS_LABEL[check.status]} ${check.name.padEnd(nameWidth)} ${check.detail}`); + } + lines.push(''); + lines.push( + report.failures === 0 && report.warnings === 0 + ? 'All checks passed.' + : `${report.failures} failure(s), ${report.warnings} warning(s).`, + ); + return lines.join('\n'); +} + +export function createDoctorCommand(deps: DoctorDeps = {}): Command { + const cmd = new Command('doctor') + .description( + 'Diagnose CLI setup: version, Node, profile, endpoint, credentials, connectivity, skill', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite doctor # run all checks (exit 1 if any fails)\n' + + ' testsprite doctor --output json # machine-readable report\n' + + ' testsprite doctor && testsprite test run # gate a command on a healthy setup', + ) + .action(async (_cmdOpts, command: Command) => { + await runDoctor(resolveCommonOptions(command), deps); + }); + + return cmd; +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + return { + profile: globals.profile ?? 'default', + output: globals.output ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const seconds = Number(raw); + if (!Number.isFinite(seconds) || seconds <= 0) { + // Match the other commands: a malformed --request-timeout is a validation + // error, not a silently-ignored default. + throw localValidationError( + 'request-timeout', + `must be a positive number of seconds (got "${raw}")`, + ); + } + return Math.round(seconds * 1000); +} + +function makeOutput(mode: OutputMode, deps: DoctorDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} diff --git a/src/index.ts b/src/index.ts index fb935c8..78010c9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; +import { createDoctorCommand } from './commands/doctor.js'; import { createDeprecatedInitCommand, createSetupCommand, @@ -89,6 +90,7 @@ program.addCommand(createProjectCommand({})); program.addCommand(createTestCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); +program.addCommand(createDoctorCommand()); // Buffer Commander error messages instead of writing immediately. The catch // block re-emits in the correct format (JSON or text) once the requested diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 231d531..323dd5c 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -661,6 +661,8 @@ Commands: Kiro, Windsurf, Copilot, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) + doctor Diagnose CLI setup: version, Node, profile, + endpoint, credentials, connectivity, skill help [command] display help for command " `; From 946f5b326453b55aaeac97dffb5bac2aa844901f Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:13:57 -0400 Subject: [PATCH 060/117] feat(cli): graceful termination signals + broken-pipe guard (#185) * feat(cli): graceful termination signals + broken-pipe guard Install handlers for SIGINT/SIGTERM/SIGHUP that print a one-line explanation (any started run keeps executing server-side; resume with `testsprite test list` or `testsprite test wait `) and exit with the conventional 128+signum code (SIGINT -> 130). Also guard EPIPE on stdout/stderr so piping to a reader that closes early (`... | head`) exits cleanly instead of dumping a raw `write EPIPE` stack. process and streams are injectable, so both are unit-tested without spawning a subprocess or sending a real signal. Fixes #75 * fix(interrupt): flush the signal message synchronously before exit A signal handler calls process.exit() immediately after writing the interrupt hint. When stderr is a pipe, an async process.stderr.write() may not flush before the process terminates, so the hint could be lost. The default stderr writer now uses fs.writeSync (best-effort, guarded against EPIPE) so the hint is reliably emitted. Added a test mocking fs.writeSync to assert the synchronous write on the default path. --- src/index.ts | 9 +++ src/lib/interrupt.test.ts | 119 +++++++++++++++++++++++++++++++++++ src/lib/interrupt.ts | 126 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 src/lib/interrupt.test.ts create mode 100644 src/lib/interrupt.ts diff --git a/src/index.ts b/src/index.ts index 78010c9..806f6e4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ import { createProjectCommand } from './commands/project.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; +import { installBrokenPipeGuard, installSignalHandlers } from './lib/interrupt.js'; import { Output, isOutputMode } from './lib/output.js'; import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; @@ -163,6 +164,14 @@ program.hook('preAction', (_thisCommand, actionCommand) => { } }); +// Clean process lifecycle: a clear message + conventional exit code on SIGINT / +// SIGTERM / SIGHUP (instead of Node's silent abrupt kill) so an interrupted +// `test run --wait` explains the run continues server-side; plus an EPIPE guard +// so piping to a reader that closes early (`| head`) exits cleanly instead of +// dumping a raw `write EPIPE` stack. +installSignalHandlers(); +installBrokenPipeGuard(); + // Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch // ignores them by default). No-op when no proxy variable is set. maybeInstallProxyAgent(); diff --git a/src/lib/interrupt.test.ts b/src/lib/interrupt.test.ts new file mode 100644 index 0000000..1fcf8eb --- /dev/null +++ b/src/lib/interrupt.test.ts @@ -0,0 +1,119 @@ +import { EventEmitter } from 'node:events'; +import { writeSync } from 'node:fs'; +import { describe, expect, it, vi } from 'vitest'; +import { + SIGINT_EXIT_CODE, + TERMINATION_EXIT_CODES, + formatInterruptMessage, + installBrokenPipeGuard, + installSignalHandlers, +} from './interrupt.js'; + +// installSignalHandlers' default stderr writes via fs.writeSync (synchronous, so +// the hint survives a piped stderr before exit); mock it to assert on that path. +vi.mock('node:fs', async importOriginal => { + const actual = (await importOriginal()) as Record; + return { ...actual, writeSync: vi.fn() }; +}); + +describe('formatInterruptMessage', () => { + it('defaults to SIGINT and explains the run continues server-side', () => { + const message = formatInterruptMessage(); + expect(message).toContain('Interrupted (SIGINT)'); + expect(message).toContain('test wait'); + expect(message).toContain('test list'); + }); + + it('names the specific signal when given one', () => { + expect(formatInterruptMessage('SIGTERM')).toContain('Interrupted (SIGTERM)'); + expect(formatInterruptMessage('SIGHUP')).toContain('Interrupted (SIGHUP)'); + }); +}); + +describe('installSignalHandlers', () => { + it('registers SIGINT, SIGTERM and SIGHUP with the conventional 128+signum exit codes', () => { + const handlers = new Map void>(); + const stderr: string[] = []; + const exit = vi.fn(); + + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + stderr: line => stderr.push(line), + exit, + }); + + expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); + + handlers.get('SIGINT')!(); + expect(exit).toHaveBeenLastCalledWith(130); + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenLastCalledWith(143); + handlers.get('SIGHUP')!(); + expect(exit).toHaveBeenLastCalledWith(129); + + // Each handler emits a leading blank line then the explanation. + expect(stderr[0]).toBe(''); + expect(stderr.join('\n')).toContain('Interrupted (SIGINT)'); + expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); + expect(stderr.join('\n')).toContain('Interrupted (SIGHUP)'); + expect(SIGINT_EXIT_CODE).toBe(130); + expect(TERMINATION_EXIT_CODES.SIGTERM).toBe(143); + expect(TERMINATION_EXIT_CODES.SIGHUP).toBe(129); + }); + + it('writes the hint synchronously via writeSync before exit (survives a piped stderr)', () => { + vi.mocked(writeSync).mockClear(); + const handlers = new Map void>(); + const exit = vi.fn(); + // No stderr dep: exercise the synchronous default path. + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + exit, + }); + handlers.get('SIGINT')!(); + expect(exit).toHaveBeenCalledWith(130); + const written = vi + .mocked(writeSync) + .mock.calls.map(call => String(call[1])) + .join(''); + expect(written).toContain('Interrupted (SIGINT)'); + }); +}); + +describe('installBrokenPipeGuard', () => { + function makeEpipe(): NodeJS.ErrnoException { + return Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }); + } + + it('exits 0 on stdout EPIPE (clean SIGPIPE-equivalent for `| head`)', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + stdout.emit('error', makeEpipe()); + expect(exit).toHaveBeenCalledWith(0); + }); + + it('re-throws a non-EPIPE stdout error instead of silently swallowing it', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + expect(() => + stdout.emit('error', Object.assign(new Error('boom'), { code: 'ENOSPC' })), + ).toThrow('boom'); + expect(exit).not.toHaveBeenCalled(); + }); + + it('swallows stderr EPIPE without exiting or throwing', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + expect(() => stderr.emit('error', makeEpipe())).not.toThrow(); + expect(exit).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/interrupt.ts b/src/lib/interrupt.ts new file mode 100644 index 0000000..cc5b4d5 --- /dev/null +++ b/src/lib/interrupt.ts @@ -0,0 +1,126 @@ +/** + * Process lifecycle hardening: graceful termination signals and broken-pipe. + * + * Termination signals: without a handler, Node terminates the process abruptly + * with no output, so a user (Ctrl+C), a CI runner or `docker stop` (SIGTERM), or + * a closed terminal/SSH session (SIGHUP) that interrupts a long + * `test run --wait` is left unsure whether the run was cancelled or is still + * executing server-side (it is: the CLI only polls; the run lives on the + * backend). The handler prints a one-line explanation plus how to resume, then + * exits with the conventional `128 + signal` code. + * + * Broken pipe: when output is piped to a reader that closes early + * (`testsprite ... | head`), the kernel raises `EPIPE` on the next stdout write. + * Node turns an `'error'` with no listener into an uncaughtException and dumps a + * raw `write EPIPE` stack (exit 1). The guard swallows it and exits 0, the + * conventional SIGPIPE-equivalent result for "the reader went away". + * + * `process` and the streams are injectable so the wiring is unit-testable + * without spawning a subprocess or sending a real signal. + */ + +import { writeSync } from 'node:fs'; + +/** + * Termination signals handled, mapped to their conventional `128 + signum` + * exit code. sourceRef: POSIX signal numbers (SIGHUP=1, SIGINT=2, SIGTERM=15). + */ +export const TERMINATION_EXIT_CODES = { + SIGINT: 130, // 128 + 2 + SIGTERM: 143, // 128 + 15 + SIGHUP: 129, // 128 + 1 +} as const; + +export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; + +/** Back-compat alias: SIGINT's conventional exit code. */ +export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; + +export function formatInterruptMessage(signal: TerminationSignal = 'SIGINT'): string { + return ( + `Interrupted (${signal}). Any run already started keeps executing on the server; ` + + 'check it with `testsprite test list` or `testsprite test wait `.' + ); +} + +export interface InterruptDeps { + /** Signal registrar. Defaults to `process.on`. */ + on?: (signal: TerminationSignal, handler: () => void) => void; + /** Line-oriented stderr writer (appends a newline). */ + stderr?: (line: string) => void; + /** Process exit. Defaults to `process.exit`. */ + exit?: (code: number) => void; +} + +/** + * Register handlers for SIGINT, SIGTERM and SIGHUP. Idempotent enough for a + * single top-level call in `index.ts`; not designed to be installed twice. + */ +export function installSignalHandlers(deps: InterruptDeps = {}): void { + const on = + deps.on ?? + ((signal: TerminationSignal, handler: () => void) => { + process.on(signal, handler); + }); + const stderr = + deps.stderr ?? + ((line: string) => { + // A signal handler calls process.exit() right after writing, which can + // truncate an async process.stderr.write() when stderr is a pipe. Write + // synchronously so the interrupt hint is flushed before the process exits. + try { + writeSync(process.stderr.fd, `${line}\n`); + } catch { + // Best-effort: if stderr is already gone (EPIPE), still exit cleanly. + } + }); + const exit = deps.exit ?? ((code: number) => process.exit(code)); + + for (const signal of Object.keys(TERMINATION_EXIT_CODES) as TerminationSignal[]) { + on(signal, () => { + // Blank line first so the message starts on its own row rather than + // trailing the progress ticker's in-place line. + stderr(''); + stderr(formatInterruptMessage(signal)); + exit(TERMINATION_EXIT_CODES[signal]); + }); + } +} + +export interface BrokenPipeDeps { + /** stdout stream. Defaults to `process.stdout`. */ + stdout?: NodeJS.EventEmitter; + /** stderr stream. Defaults to `process.stderr`. */ + stderr?: NodeJS.EventEmitter; + /** Process exit. Defaults to `process.exit`. */ + exit?: (code: number) => void; +} + +/** + * Guard against `EPIPE` on stdout/stderr so piping to a reader that closes + * early (`testsprite ... | head`) exits cleanly instead of crashing with an + * unhandled `write EPIPE` stack. Only `EPIPE` is swallowed; any other stream + * error is left to surface normally. + */ +export function installBrokenPipeGuard(deps: BrokenPipeDeps = {}): void { + const stdout = deps.stdout ?? process.stdout; + const stderr = deps.stderr ?? process.stderr; + const exit = deps.exit ?? ((code: number) => process.exit(code)); + + stdout.on('error', (error: NodeJS.ErrnoException) => { + // Reader went away (`| head`, `| less` then q): exit cleanly like SIGPIPE + // rather than dumping an unhandled `write EPIPE` stack. Any other stdout + // error is a genuine, actionable failure, so re-throw it (Node's default). + if (error.code === 'EPIPE') { + exit(0); + return; + } + throw error; + }); + stderr.on('error', (error: NodeJS.ErrnoException) => { + // stderr closed: nothing can be reported over it, so swallow EPIPE. Any + // other error re-throws so a genuine failure is not silently hidden. + if (error.code === 'EPIPE') return; + throw error; + }); +} From 3305dfa57ab258b11311d15197a80dd9eb654911 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:14:29 -0400 Subject: [PATCH 061/117] feat(test): add "test scaffold" to emit a schema-correct starter plan or backend skeleton (#180) --- src/commands/test.test.ts | 90 +++++++++++ src/commands/test.ts | 145 +++++++++++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 4 + 3 files changed, 238 insertions(+), 1 deletion(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 0368e06..6ae831c 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -35,6 +35,7 @@ import { runList, runPlanPut, runResult, + runScaffold, runSteps, runTestWaitMany, runUpdate, @@ -134,6 +135,7 @@ describe('createTestCommand — surface', () => { 'rerun', 'result', 'run', + 'scaffold', 'steps', 'update', 'wait', @@ -2325,6 +2327,94 @@ describe('runCodePut', () => { }); }); +describe('runScaffold', () => { + it('frontend scaffold is a valid CliPlanInput and prints as JSON on stdout', async () => { + const out: string[] = []; + const result = await runScaffold( + { profile: 'default', output: 'text', debug: false, scaffoldType: 'frontend', force: false }, + { stdout: line => out.push(line), stderr: () => undefined, env: {} }, + ); + const plan = result as { projectId: string; type: string; planSteps: Array<{ type: string }> }; + expect(plan.type).toBe('frontend'); + // Placeholder project id when TESTSPRITE_PROJECT_ID is unset. + expect(plan.projectId).toContain('testsprite project list'); + expect(plan.planSteps.length).toBeGreaterThanOrEqual(2); + // Every emitted step type must come from the real enum (no drift). + for (const step of plan.planSteps) expect(['action', 'assertion']).toContain(step.type); + // At least one assertion step so the scaffold is a meaningful test. + expect(plan.planSteps.some(step => step.type === 'assertion')).toBe(true); + // stdout body parses back to the same plan (`> plan.json` works). + expect(JSON.parse(out.join('\n'))).toEqual(plan); + }); + + it('pre-fills projectId from TESTSPRITE_PROJECT_ID when set', async () => { + const result = await runScaffold( + { profile: 'default', output: 'json', debug: false, scaffoldType: 'frontend', force: false }, + { + stdout: () => undefined, + stderr: () => undefined, + env: { TESTSPRITE_PROJECT_ID: 'project_env' }, + }, + ); + expect((result as { projectId: string }).projectId).toBe('project_env'); + }); + + it('backend scaffold defines a requests test with a status assertion AND calls it', async () => { + const out: string[] = []; + const result = await runScaffold( + { profile: 'default', output: 'text', debug: false, scaffoldType: 'backend', force: false }, + { stdout: line => out.push(line), stderr: () => undefined, env: {} }, + ); + const code = (result as { code: string }).code; + expect(code).toContain('import requests'); + expect(code).toContain('assert response.status_code == 200'); + // The onboarding rule: the function must be CALLED, not just defined. + expect(code).toContain('\ntest_health_endpoint()'); + expect(out.join('\n')).toContain('import requests'); + }); + + it('--out writes the file and refuses to overwrite without --force', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-')); + const target = join(dir, 'plan.json'); + const opts = { + profile: 'default', + output: 'text', + debug: false, + scaffoldType: 'frontend', + out: target, + force: false, + } as const; + const deps = { stdout: () => undefined, stderr: () => undefined, env: {} }; + await runScaffold({ ...opts }, deps); + const written = JSON.parse(readFileSync(target, 'utf8')) as { type: string }; + expect(written.type).toBe('frontend'); + // Second run without --force must not clobber the (possibly edited) file. + await expect(runScaffold({ ...opts }, deps)).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + // --force overwrites. + await expect(runScaffold({ ...opts, force: true }, deps)).resolves.toBeDefined(); + }); + + it('--out pointing at an existing path (here a directory) rejects without --force', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-dir-')); + await expect( + runScaffold( + { + profile: 'default', + output: 'text', + debug: false, + scaffoldType: 'frontend', + out: dir, + force: false, + }, + { stdout: () => undefined, stderr: () => undefined, env: {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + describe('runSteps', () => { it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index 57d90f5..1c6edd8 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1,4 +1,11 @@ -import { createWriteStream, readFileSync, readdirSync, statSync, type WriteStream } from 'node:fs'; +import { + createWriteStream, + existsSync, + readFileSync, + readdirSync, + statSync, + type WriteStream, +} from 'node:fs'; import { rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path'; import { randomUUID } from 'node:crypto'; @@ -4000,6 +4007,114 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const env = deps.env ?? process.env; + // Pre-fill the project id from TESTSPRITE_PROJECT_ID when the caller's + // environment carries one; otherwise a clearly-marked placeholder the user + // swaps after running `testsprite project list`. + const projectId = + typeof env.TESTSPRITE_PROJECT_ID === 'string' && env.TESTSPRITE_PROJECT_ID.length > 0 + ? env.TESTSPRITE_PROJECT_ID + : ''; + + let payload: CliPlanInput | CliBackendScaffold; + let body: string; + if (opts.scaffoldType === 'frontend') { + const plan: CliPlanInput = { + projectId, + type: 'frontend', + name: 'My first frontend test', + description: 'Replace with one sentence describing what this test verifies.', + priority: 'p2', + planSteps: [ + { + type: 'action', + description: 'Navigate to /login and sign in with a seeded test account', + }, + { type: 'action', description: 'Open the first product page and click "Add to cart"' }, + { type: 'assertion', description: 'Assert that the cart badge shows 1 item' }, + ], + }; + payload = plan; + body = `${JSON.stringify(plan, null, 2)}\n`; + } else { + const code = [ + 'import requests', + '', + '# Replace with your API base URL (must be reachable from the internet).', + 'BASE_URL = "https://staging.example.com"', + '', + '', + 'def test_health_endpoint() -> None:', + ' response = requests.get(f"{BASE_URL}/health", timeout=30)', + ' assert response.status_code == 200, f"expected 200, got {response.status_code}"', + '', + '', + '# The test function MUST be called: TestSprite executes this file top to', + '# bottom, so a defined-but-never-called function would pass vacuously.', + 'test_health_endpoint()', + '', + ].join('\n'); + payload = { type: 'backend', language: 'python', code }; + body = code; + } + + if (opts.out !== undefined) { + const resolved = isAbsolute(opts.out) ? opts.out : resolve(process.cwd(), opts.out); + // Never clobber silently: scaffolds are starting points the user edits, so + // an accidental re-run must not erase their work. --force opts in. + if (!opts.force && existsSync(resolved)) { + throw localValidationError('out', `already exists: ${resolved}. Pass --force to overwrite`); + } + const sink = openOutputFile(opts.out); // reuses the directory/parent guards + const fileOut = makeFileOutput(opts.output, sink); + await fileOut.writeChunk(body); + await closeOutputFile(sink, true); + stderrFn(`Scaffold written to ${resolved}`); + return payload; + } + + // No --out: the scaffold body IS the stdout payload (`> plan.json` works). + out.print(payload, () => body.trimEnd()); + return payload; +} + export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -8007,6 +8122,34 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('scaffold') + .description( + 'Emit a schema-correct starter test definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials.', + ) + .option('--type ', 'frontend|backend (default: frontend)') + .option('--out ', 'write the scaffold to a file instead of stdout') + .option('--force', 'overwrite an existing --out file', false) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite test scaffold > first-test.plan.json\n' + + ' testsprite test scaffold --type backend --out tests/health.py\n' + + ' testsprite test scaffold --out plan.json # then edit, and create with --plan-from plan.json', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: ScaffoldFlagOpts, command: Command) => { + await runScaffold( + { + ...resolveCommonOptions(command), + scaffoldType: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES) ?? 'frontend', + out: cmdOpts.out, + force: cmdOpts.force === true, + }, + deps, + ); + }); + test .command('steps ') .description( diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 323dd5c..fcc3d85 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -201,6 +201,10 @@ Commands: (--plan-from, FE-only, M3.2 piece-5) create-batch [options] Create multiple FE tests from a JSONL of plan specs (FE-only) + scaffold [options] Emit a schema-correct starter test + definition (frontend plan JSON by + default, or a backend Python skeleton). + Pure-local: no network, no credentials. steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) From 3d65ff5a0e25399a7b87e8c75b6f8eb90f0ef2f8 Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Thu, 9 Jul 2026 15:29:24 -0700 Subject: [PATCH 062/117] release: v0.3.0 Private-Snapshot-RevId: d22d9f18f5d9712921097d5829c28d056e3ffd31 --- .gitattributes | 6 + .github/PULL_REQUEST_TEMPLATE.md | 4 +- .github/workflows/ci-nudge.yml | 129 ++++++ .github/workflows/issue-triage.yml | 23 +- .github/workflows/pr-triage.yml | 189 +++++++++ .github/workflows/stale.yml | 64 +++ CHANGELOG.md | 47 ++- CONTRIBUTING.md | 48 ++- DOCUMENTATION.md | 190 +++++++-- README.md | 52 ++- package.json | 2 +- skills/testsprite-onboard.skill.md | 21 + skills/testsprite-verify.skill.md | 33 ++ src/commands/agent.test.ts | 50 ++- src/commands/agent.ts | 21 +- src/commands/init.test.ts | 17 +- src/commands/init.ts | 27 +- src/commands/project.test.ts | 268 ++++++++++++- src/commands/project.ts | 376 +++++++++++++++++- src/commands/test.test.ts | 72 ++++ src/commands/test.ts | 87 +++- src/commands/usage.ts | 6 +- src/index.ts | 2 +- src/lib/agent-targets.test.ts | 4 +- src/lib/bundle.test.ts | 6 +- src/lib/credentials.test.ts | 14 +- src/lib/http.ts | 6 +- src/lib/skill-nudge.test.ts | 18 +- src/version.ts | 2 +- test/__snapshots__/help.snapshot.test.ts.snap | 52 ++- test/cli.subprocess.test.ts | 12 +- test/helpers/hermetic-env.ts | 41 ++ vitest.config.ts | 3 + 33 files changed, 1696 insertions(+), 196 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/ci-nudge.yml create mode 100644 .github/workflows/pr-triage.yml create mode 100644 .github/workflows/stale.yml create mode 100644 test/helpers/hermetic-env.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..468c8ed --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Check out all text files with LF on every platform. Tests compare bytes +# from checked-out files (skill templates, snapshots); a CRLF working tree +# (core.autocrlf on Windows) broke those comparisons. +* text=auto eol=lf + +*.png binary diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c490f9b..1c165f6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,7 +12,9 @@ agree on the approach before you invest time — see CONTRIBUTING.md. ## Related issue - + ## Type of change diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml new file mode 100644 index 0000000..d01ede3 --- /dev/null +++ b/.github/workflows/ci-nudge.yml @@ -0,0 +1,129 @@ +# "Fix this and we merge" nudge (InsForge `agent-zhang-beihai` pattern, part 3). +# When a PR's CI finishes red, posts ONE sticky comment listing exactly which +# jobs failed and the local one-liner that reproduces/fixes each (the automated +# version of the hand-written "run `npm run format` and you're green" review +# comments). The comment flips to a green confirmation once all checks pass. +# +# Runs in base-repo context via workflow_run — no PR code is checked out, so +# fork PRs are safe. State is recomputed from check-runs each time, so the two +# CI workflows (CI + Test Coverage) can complete in any order. +# +# Bot identity: same App-token-first / GITHUB_TOKEN-fallback pattern as +# pr-triage.yml (the App needs the "Pull requests: Read & write" permission to +# post as testsprite-hob[bot]; until then comments come from github-actions[bot]). +name: CI failure nudge + +on: + workflow_run: + workflows: ['CI', 'Test Coverage'] + types: [completed] + +permissions: + checks: read # read the head SHA's check-run state + pull-requests: write + issues: write # PR comments ride the issues API + +env: + MARKER: '' + +jobs: + nudge: + # Public repo only; PR-triggered runs only (pushes to main have no PR to nudge). + if: >- + github.repository == 'TestSprite/testsprite-cli' && + github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + with: + script: | + const { owner, repo } = context.repo; + const run = context.payload.workflow_run; + const marker = process.env.MARKER; + + const appClient = process.env.APP_TOKEN + ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + : null; + async function write(fn) { + if (appClient) { + try { return await fn(appClient.rest); } + catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } + } + return await fn(github.rest); + } + + // Resolve the PR for this run. `workflow_run.pull_requests` is empty + // for fork PRs, so fall back to the commit→PRs lookup. + let pr = (run.pull_requests || [])[0]; + if (!pr) { + const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit( + { owner, repo, commit_sha: run.head_sha }); + pr = data.find(p => p.state === 'open'); + } else { + pr = (await github.rest.pulls.get({ owner, repo, pull_number: pr.number })).data; + } + if (!pr || pr.state !== 'open' || pr.user.type === 'Bot') return; + + // Job name → the local command that reproduces/fixes it. Also the + // allowlist of CI jobs this nudge watches — keep in sync with ci.yml. + const FIX = { + 'Lint & Format': 'run `npm run lint:fix && npm run format`, then commit', + 'Typecheck': 'run `npm run typecheck` and fix the reported type errors', + 'Unit Tests': 'run `npm test` and fix the failing tests', + 'Build': 'run `npm run build` and fix the compile errors', + 'Local E2E Tests': 'run `npm run test:e2e` (it builds first)', + 'Coverage (>= 80%)': 'run `npm run test:coverage` — new code needs tests until every metric is back at 80%', + }; + + // Recompute full CI state from this SHA's check runs, narrowed to the + // CI jobs above — other workflows (e.g. pr-triage) also attach + // github-actions check runs to the PR head and must not count here. + const checks = await github.paginate(github.rest.checks.listForRef, + { owner, repo, ref: run.head_sha, per_page: 100 }); + const ours = checks.filter(c => c.app && c.app.slug === 'github-actions' && FIX[c.name]); + const failing = ours.filter(c => ['failure', 'timed_out'].includes(c.conclusion)); + const pending = ours.filter(c => c.status !== 'completed'); + + const comments = await github.paginate(github.rest.issues.listComments, + { owner, repo, issue_number: pr.number, per_page: 100 }); + const sticky = comments.find(c => (c.body || '').includes(marker)); + + if (failing.length === 0) { + // Only speak up on success if we previously flagged a failure, and + // only once everything has actually finished. + if (sticky && pending.length === 0 && !sticky.body.includes('all green')) { + await write(rest => rest.issues.updateComment({ owner, repo, comment_id: sticky.id, + body: `${marker}\n✅ CI is **all green** now — thanks, @${pr.user.login}!` })); + } + return; + } + + const lines = failing + .sort((a, b) => a.name.localeCompare(b.name)) + .map(c => { + const fix = FIX[c.name] || 'see the logs for details'; + return `- **${c.name}** — ${fix} ([logs](${c.html_url}))`; + }); + const body = `${marker}\nThanks, @${pr.user.login}! CI is red on this PR — ` + + `here's what failed and how to reproduce it locally:\n\n${lines.join('\n')}\n\n` + + `Everything runs on Node 22 after \`npm ci\`. Push a fix and this comment ` + + `flips green automatically once all checks pass.`; + + if (sticky) { + if (sticky.body !== body) { + await write(rest => rest.issues.updateComment( + { owner, repo, comment_id: sticky.id, body })); + } + } else { + await write(rest => rest.issues.createComment( + { owner, repo, issue_number: pr.number, body })); + } diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 63323c0..6b6ce62 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -1,13 +1,15 @@ # P1-2 issue auto-assign + 3-slot-cap bot (InsForge `agent-zhang-beihai` pattern). -# Posts as the `alfheim-agent` GitHub App ⇒ `alfheim-agent[bot]`. +# Posts as the `testsprite-hob` GitHub App ⇒ `testsprite-hob[bot]`. # # Setup (one-time, by an org admin): -# 1. Create a GitHub App named `alfheim-agent` (org Settings → Developer settings → +# 1. Create a GitHub App named `testsprite-hob` (org Settings → Developer settings → # GitHub Apps → New). Permissions: Issues = Read & write, Metadata = Read. No webhook. # Generate a private key; install the App on the public testsprite-cli repo. # 2. Add two repo (or org) secrets: -# ALFHEIM_AGENT_APP_ID = the App's numeric App ID -# ALFHEIM_AGENT_PRIVATE_KEY = the App's .pem private key (full contents) +# TESTSPRITE_HOB_APP_ID = the App's numeric App ID +# TESTSPRITE_HOB_PRIVATE_KEY = the App's .pem private key (full contents) +# (The ALFHEIM_AGENT_* fallbacks are this App's original secret names from +# before it was renamed — same App ID + key; either naming works.) # Until those exist, the bot gracefully falls back to github-actions[bot] (still works). # # Fires on each new issue comment; assigns the commenter when they claim an issue @@ -30,23 +32,26 @@ env: jobs: triage: # issues only (issue_comment also fires on PRs), and never react to a bot's own - # comment — incl. our own alfheim-agent[bot], which (unlike GITHUB_TOKEN) would + # comment — incl. our own testsprite-hob[bot], which (unlike GITHUB_TOKEN) would # otherwise re-trigger this workflow. `type == 'Bot'` covers both bot identities. if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' }} runs-on: ubuntu-latest steps: - # Mint a token for the alfheim-agent App so comments post as alfheim-agent[bot]. + # Mint a token for the testsprite-hob App so comments post as testsprite-hob[bot]. # continue-on-error: until the App + secrets exist this no-ops and we fall back below. - id: app-token continue-on-error: true uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - app-id: ${{ secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # Scope the minted token to what this job uses (issues API only) instead + # of inheriting the App's full installation permissions. + permission-issues: write - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 with: - # App token when available (→ alfheim-agent[bot]); else the default + # App token when available (→ testsprite-hob[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. github-token: ${{ steps.app-token.outputs.token || github.token }} script: | diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml new file mode 100644 index 0000000..a9cd23d --- /dev/null +++ b/.github/workflows/pr-triage.yml @@ -0,0 +1,189 @@ +# PR-side twin of issue-triage.yml (InsForge `agent-zhang-beihai` pattern, part 2). +# Enforces the issue-first workflow on community PRs: every non-docs PR must +# carry a closing link ("Closes #123") to an issue that is ASSIGNED to the PR +# author (claimed via `/assign`, which issue-triage.yml handles). Violations +# get a `needs-issue` label + a sticky comment, and — for PRs opened after +# GATE_SINCE — a failing check, so unclaimed work is visibly not review-ready. +# PRs opened before GATE_SINCE are grandfathered: nudge only, never a red check. +# +# Assignment happens on the ISSUE, so it cannot re-trigger this PR workflow; +# the comment tells the contributor to edit the PR description or push a +# commit (`edited` / `synchronize`) to re-run the gate. +# +# Runs with NO checkout — metadata-only, so it is safe under pull_request_target +# (which is required for fork PRs to get a write-capable token). +# +# Bot identity: posts as `testsprite-hob[bot]` when the App token works. The +# App currently has Issues R/W + Metadata only — commenting/labeling a PULL +# REQUEST needs the "Pull requests: Read & write" App permission (issues-API +# endpoints are permission-checked by target type). Until an org admin adds +# that permission and re-approves the installation, every call gracefully falls +# back to the default GITHUB_TOKEN and posts as github-actions[bot]. +# Secrets: TESTSPRITE_HOB_* preferred; the ALFHEIM_AGENT_* fallbacks are the +# original names from before the App was renamed (same App ID + key). +name: PR triage (issue-link gate) + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +permissions: + pull-requests: write # add/remove the needs-issue label + issues: write # create/update the nudge comment (PR comments ride the issues API) + +env: + LABEL: 'needs-issue' + MARKER: '' + # PRs created before this instant are grandfathered (nudge, no failing check). + GATE_SINCE: '2026-07-04T00:00:00Z' + +jobs: + gate: + # Public repo only — this file also lives in the private mirror, where PRs + # are internal work that never links public issues. Skip bot authors + # (dependabot etc.), maintainers, and docs-only changes (CONTRIBUTING + # exempts docs/small fixes from the issue-first ask). + if: >- + github.repository == 'TestSprite/testsprite-cli' && + github.event.pull_request.state == 'open' && + github.event.pull_request.user.type != 'Bot' && + !contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) && + !startsWith(github.event.pull_request.title, 'docs') + runs-on: ubuntu-latest + steps: + # Mint an App token so actions post as testsprite-hob[bot]. Until the App + + # secrets + Pull-requests permission exist this no-ops / gets 403 and the + # script below falls back to the default token per-call. + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + with: + # `github` client = default GITHUB_TOKEN: used for all reads (the App + # can't read PRs without the Pull-requests permission) and as the + # write fallback. App client (when mintable) is preferred for writes. + script: | + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + const label = process.env.LABEL; + const marker = process.env.MARKER; + const author = pr.user.login; + + const appClient = process.env.APP_TOKEN + ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + : null; + // Prefer the App identity; fall back to github-actions[bot] when the + // App is absent or lacks the Pull-requests permission (403/404). + async function write(fn) { + if (appClient) { + try { return await fn(appClient.rest); } + catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } + } + return await fn(github.rest); + } + + // Linked issues: GitHub-computed closing references carry number + + // assignees in one query. + const gql = await github.graphql( + `query($owner:String!,$repo:String!,$num:Int!){ + repository(owner:$owner,name:$repo){ + pullRequest(number:$num){ + closingIssuesReferences(first:10){ + nodes{ number assignees(first:10){ nodes{ login } } } + } + } + } + }`, { owner, repo, num: pr.number }); + const linkedIssues = gql.repository.pullRequest.closingIssuesReferences.nodes + .map(n => ({ number: n.number, assignees: n.assignees.nodes.map(a => a.login) })); + + // Body-text fallback for the brief window before GitHub computes the + // link (same-repo bare "#N" references only). + const bodyRefs = [...(pr.body || '') + .matchAll(/(close[sd]?|fix(e[sd])?|resolve[sd]?)\s*:?\s+#(\d+)/gi)] + .map(m => Number(m[3])) + .filter(n => !linkedIssues.some(i => i.number === n)) + .slice(0, 5); + for (const num of bodyRefs) { + try { + const { data } = await github.rest.issues.get({ owner, repo, issue_number: num }); + if (data.pull_request) continue; // "#N" pointed at a PR, not an issue + linkedIssues.push({ number: num, assignees: (data.assignees || []).map(a => a.login) }); + } catch (e) { /* unknown number — ignore */ } + } + + const linked = linkedIssues.length > 0; + const assignedToAuthor = linkedIssues.some(i => i.assignees.includes(author)); + const state = assignedToAuthor ? 'ok' : (linked ? 'unassigned' : 'unlinked'); + + const hasLabel = (pr.labels || []).some(l => l.name === label); + const comments = await github.paginate(github.rest.issues.listComments, + { owner, repo, issue_number: pr.number, per_page: 100 }); + const nudge = comments.find(c => (c.body || '').includes(marker)); + const stateLine = ``; + + const contributingUrl = + `https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#contribution-model`; + const rerunHint = 'After fixing it, edit the PR description or push a commit to re-run this check.'; + + let body; + if (state === 'ok') { + body = `${marker}\n${stateLine}\n` + + `✅ This PR is linked to an issue assigned to @${author} — thanks! ` + + `The \`${label}\` label has been removed.`; + } else if (state === 'unassigned') { + const list = linkedIssues.map(i => { + const holders = i.assignees.filter(a => a !== author); + return `#${i.number}` + (holders.length ? ` (currently assigned to @${holders.join(', @')})` : ' (unassigned)'); + }).join(', '); + body = `${marker}\n${stateLine}\n` + + `Thanks for the PR, @${author}! It links an issue, but that issue isn't assigned to you yet: ${list}. ` + + `Per our workflow, **claim the issue first by commenting \`/assign\` on it** (the triage bot assigns you automatically). ` + + `If it's already assigned to someone else, please coordinate with them or pick another issue — ` + + `unclaimed-issue PRs are not reviewed. ${rerunHint} ` + + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; + } else { + body = `${marker}\n${stateLine}\n` + + `Thanks for the PR, @${author}! A quick note on our workflow: for **features and behavior changes** ` + + `we require contributors to **open an issue first, claim it by commenting \`/assign\` on the issue, ` + + `then submit a PR that links it** (e.g. \`Closes #123\`). This PR isn't linked to any issue yet, ` + + `so it is not review-ready. ${rerunHint} ` + + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; + } + + // Sticky comment: create once, update when the state changes. + if (!nudge) { + if (state !== 'ok') { + await write(rest => rest.issues.createComment( + { owner, repo, issue_number: pr.number, body })); + } + } else if (!nudge.body.includes(stateLine)) { + await write(rest => rest.issues.updateComment( + { owner, repo, comment_id: nudge.id, body })); + } + + // Label tracks the gate state. + if (state === 'ok' && hasLabel) { + await write(rest => rest.issues.removeLabel( + { owner, repo, issue_number: pr.number, name: label })).catch(() => {}); + } + if (state !== 'ok' && !hasLabel) { + await write(rest => rest.issues.addLabels( + { owner, repo, issue_number: pr.number, labels: [label] })); + } + + // Hard gate for PRs opened after the cutoff; older PRs are nudged only. + const gated = new Date(pr.created_at) >= new Date(process.env.GATE_SINCE); + if (state !== 'ok' && gated) { + core.setFailed(state === 'unlinked' + ? 'No closing-linked issue. Open/claim an issue, add "Closes #" to the PR description, then re-run.' + : 'Linked issue is not assigned to the PR author. Comment /assign on the issue, then re-run.'); + } else if (state !== 'ok') { + core.notice(`Grandfathered PR (opened before ${process.env.GATE_SINCE}): gate not enforced, nudge only.`); + } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..fbaa607 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,64 @@ +# P1-3 — stale-bot: nudge then close inactive issues / PRs (actions/stale). +# +# Generous windows for an open-source repo (first-response SLA is 5 business days, +# see CONTRIBUTING). Activity removes the `stale` label automatically, so a single +# reply resets the clock. Newcomer- and security-relevant work is exempt from +# closing. `good first issue` / `help wanted` stay open for whoever picks them up. +# +# This is a SYNCED asset (ships to the public mirror). It is scheduled, so unlike +# the event-gated triage bot it is fenced to the PUBLIC repo with a repository +# guard — on private atlas it is a no-op (no nagging internal issues/PRs). +name: Stale + +on: + schedule: + - cron: '30 1 * * *' # daily 01:30 UTC + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + stale: + if: ${{ github.repository == 'TestSprite/testsprite-cli' }} + runs-on: ubuntu-latest + permissions: + issues: write # comment + (un)label + close stale issues + pull-requests: write # comment + (un)label + close stale PRs + steps: + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 + with: + # ── issues ────────────────────────────────────────────────────── + days-before-issue-stale: 60 + days-before-issue-close: 14 + stale-issue-label: stale + stale-issue-message: > + This issue has had no activity for 60 days, so it's been marked + `stale`. If it's still relevant, just leave a comment (or remove the + `stale` label) and we'll keep it open — otherwise it will be closed + in 14 days. Thanks for helping us keep the tracker tidy! + close-issue-message: > + Closing as `stale` after no activity. This isn't a judgement on the + idea — please reopen or open a fresh issue if it's still relevant. + + # ── pull requests (more grace — external contributors may be slow) ─ + days-before-pr-stale: 45 + days-before-pr-close: 21 + stale-pr-label: stale + stale-pr-message: > + This PR has had no activity for 45 days, so it's been marked `stale`. + Push a commit or leave a comment to keep it open — otherwise it will + be closed in 21 days. We'd still love to merge it; ping a maintainer + if you're blocked on a review. + close-pr-message: > + Closing as `stale` after no activity. Reopen any time you can pick it + back up — your work isn't lost. + + # ── shared behaviour ──────────────────────────────────────────── + exempt-issue-labels: 'pinned,security,in-progress,good first issue,help wanted,hackathon' + exempt-pr-labels: 'pinned,security,in-progress,hackathon' + exempt-draft-pr: true + remove-stale-when-updated: true + ascending: true # oldest first — fairest under the per-run op cap + operations-per-run: 60 + enable-statistics: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f1a0a8..e78347e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,48 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +## [0.3.0] - 2026-07-08 + ### Added -- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. -- **`testsprite test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs `, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky --runs 5 || exit 1`). Flags: `--runs ` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout ` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits. +- **`testsprite doctor`** — one-command environment diagnostic that checks your Node version, credentials, endpoint reachability, and installed agent skills, and reports what's misconfigured. +- **`test scaffold`** — emit a schema-correct starter plan (frontend) or a backend test skeleton to bootstrap a new test without hand-writing the JSON. +- **`test lint`** — offline validator for plan / steps files; catches malformed test definitions before they are sent to the server. +- **`test diff `** — compare two runs of the same test to isolate what changed between a passing and a failing run. +- **`test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs`, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal off (strict verbatim) so a nondeterministic pass/fail can't be masked. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness. Flags: `--runs` (1–10), `--until-fail`, `--timeout`, `--output json`. +- **JUnit XML report export for batch runs.** `test run --all` and batch `test rerun` accept `--report junit --report-file ` to write a CI-friendly XML sidecar after `--wait` polling completes. The report is written even when the batch exits non-zero; `--output json` is unchanged; `--dry-run` writes a canned sample without network calls. +- **`test wait` is now variadic** — pass several run ids to attach to and poll multiple runs in a single invocation. +- **`agent status`** — report which TestSprite skills are installed for each agent target and whether they are current; installed skills are now stamped with a version/hash marker. +- **New `agent install` targets:** GitHub Copilot, Windsurf, and Kiro (experimental), alongside the existing Claude / Cursor / Cline / Codex / Antigravity targets. +- **`project credential` / `project auto-auth`** — configure a project's backend credentials (static credential, free) or a recurring auto-auth token (Pro) from the CLI, with surfaced auth warnings and managed-credential guidance. +- **Proxy support** — the CLI now honors `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` for use behind corporate and CI proxies. +- **`NO_COLOR` support** — colored output is suppressed when `NO_COLOR` is set, per no-color.org. +- **"New version available" notice** — a non-blocking, 24h-cached npm version check prints an upgrade hint on stderr. Opt out with the documented env var; automatically silenced in CI and under `--output json`. + +### Changed + +- **Node.js 20.19+, 22.13+, or 24+ is now the minimum supported runtime.** The CLI checks the running Node version at startup and exits with a clear message on an unsupported version; builds and CI run against Node 20 and 22. +- **Graceful shutdown** — the CLI handles termination signals cleanly and guards against broken-pipe (`EPIPE`) errors when its output is piped to a closing consumer (e.g. `| head`). +- **Interactive prompts and preamble now go to stderr**, keeping stdout pure for machine consumers even in interactive mode. +- **Empty environment variables are treated as unset** when resolving config, so `TESTSPRITE_API_URL=` no longer overrides the built-in default with an empty string. +- `agent install` defaults `--target` to `claude` in non-interactive / CI contexts (matching `setup`). +- The `usage` command no longer implies backend test runs are free. +- `setup`'s "Next steps" guidance no longer suggests `test list` before any project exists. + +### Fixed + +- **Timeouts & polling:** `RequestTimeoutError` is now classified as a timeout in the `--all --wait` fan-out; per-attempt timeout timers are cleared so they can't fire late; `run --all --wait` no longer polls still-queued runs past the shared deadline; a partial result is emitted on stdout when `run --wait` / `test wait` times out (so a redirected file is never zero-byte). +- **Batch rerun:** the exit code is preserved and auth errors escalate correctly; explicit ids combined with `--all` — or `--status` / `--skip-terminal` without `--all` — are rejected with a clear validation error; auto-minted idempotency keys are surfaced under `--output json`. +- **HTTP:** non-JSON `200` responses map to a typed error envelope instead of crashing the parser. +- **Failure bundles / artifacts:** artifact downloads retry on transient errors and guard the default run-id path; the `--out` directory no longer sweeps unrelated pre-existing files (data-loss fix); run-scoped per-step error text and step type are surfaced. +- **Input validation (fail fast, before any network call):** malformed API keys, invalid `--request-timeout`, directory `--code-file` / `--out` paths, blank or whitespace-only `--name` (test and project create/update), blank inline project passwords, fractional pagination flags / page sizes, and `--since` overflow are all rejected up front with `VALIDATION_ERROR` rather than crashing or failing late server-side. `--output` is validated uniformly across all command groups. +- **Setup / auth:** the endpoint is validated before the key check; the typed API-error envelope is preserved when key verification fails; the per-request timeout is honored during `configure`. +- **Misc:** cursor pagination no longer drops empty pages; trailing-dot hostnames are treated as loopback by the local-target guard; buffered input is preserved between interactive prompts; the Codex managed-section skill check requires a complete section; `code get` strips a leading BOM and rejects an empty `--out`. + +### Security + +- **INI injection:** CR/LF characters are stripped from credential values before they are written to `~/.testsprite/credentials`. +- **Symlink fail-close:** the own-file `agent install` path applies its symlink containment guard under `--dry-run` as well, so a planted symlink cannot place or clobber files outside `--dir`. ## [0.2.0] - 2026-06-29 @@ -149,6 +187,9 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for - Commander `help [command]` exits 0 (previously exited 5 on `test help` / `project help`). -[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...HEAD +[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.2...v0.2.0 +[0.1.2]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/TestSprite/testsprite-cli/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2110f0b..bed91fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,14 +20,54 @@ Discussions**, so the issue tracker stays a clean, actionable list. ## Contribution model -- **Small fixes and improvements:** just open a pull request. No issue required. -- **Large or breaking changes** (new commands, changed flags/output, new - dependencies, refactors): **open an issue first** to discuss the design. This - avoids wasted work on something we'd ask you to rework or that's out of scope. +- **Docs and small fixes** (typos, doc corrections, comment-only changes): + just open a pull request. No issue required (linking one is still + appreciated). +- **Features and behavior changes** (new commands or flags, changed output, + new dependencies, refactors) follow **issue-first**: + 1. Find an existing issue, or open a new one describing the change. + 2. Claim it by commenting `/assign` on the issue — the literal slash + command on its own line. The triage bot assigns you automatically; + free-text requests ("can I take this?") are **not** detected. + 3. If the issue is new, wait for triage — we check proposals against + [VISION.md](./VISION.md) and the [standing policies](#standing-scope-policies) + below before any code is written, so you don't invest in something + we'd ask you to rework or decline. + 4. Open your PR with a closing link (e.g. `Closes #123`) in the + description. +- **PR gate:** a bot checks every non-docs community PR for a closing-linked + issue that is **assigned to the PR author**. PRs that don't meet this get + the `needs-issue` label and a failing `PR triage` check, and **are not + reviewed** until it's fixed — file or claim the issue, add the closing + link, then edit the PR description (or push a commit) to re-run the check. +- Suspected **security vulnerabilities** are the exception to "file an + issue": report them privately per [SECURITY.md](./SECURITY.md) instead. - We **do** accept community code contributions — this is an actively maintained open-source CLI, not a read-only distribution mirror. - See [VISION.md](./VISION.md) for what is in and out of scope. +### Standing scope policies + +Pre-decided policies, so proposals don't have to relitigate them: + +- **Runtime dependencies are budgeted.** The CLI ships with a deliberately + tiny runtime dependency set (`commander`, `valibot`, plus `undici` — + approved for `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` support). Any new + runtime dependency needs explicit maintainer sign-off **in the issue, + before the PR**. Utility modules land only together with the consumer + that uses them — standalone libraries are declined. +- **`agent install` targets.** Shipped: `claude`, `antigravity`, `cursor`, + `cline`, `codex`, `kiro`, `windsurf`, `copilot`. Accepted and in progress: + `gemini`. + A proposal for a new target needs (1) the editor's official rules/skill + file mechanism, documented, and (2) the proposer prepared to maintain the + target going forward. +- **Outbound network calls.** The CLI talks only to the configured + TestSprite API endpoint. The one approved exception is an opt-out-able + npm registry version check (at most once per 24h, carrying nothing but + the package name, fully silenced by its env opt-out and in CI/JSON/dry-run + modes or when stderr is not a TTY). Any other outbound call is out of scope. + We aim to give every issue and PR a **first response within 5 business days** (best-effort). If something has gone quiet longer than that, a polite nudge on the thread or in Discord is welcome. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 6b798ef..2f6cd6c 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -15,6 +15,7 @@ The full reference for the TestSprite CLI: install verification, manual setup, e - [Read commands](#read-commands) - [Write commands](#write-commands) - [Run commands](#run-commands) + - [Account & diagnostics](#account--diagnostics) - [Configuration](#configuration) - [Output & scripting](#output--scripting) - [Exit codes](#exit-codes) @@ -117,10 +118,15 @@ testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.m testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md testsprite agent list # list all 8 targets with status + mode + path +testsprite agent status # check installed skills against this CLI version ``` Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). +Omitting `--target` in a non-interactive shell (CI, agent subprocess) defaults to `claude` with an `[info]` note on stderr; in a terminal the CLI prompts (empty answer = `claude`). + +`agent status` checks every installed skill file against the current CLI version and reports one of `ok`, `stale`, `modified`, `unmarked`, `absent`, or `corrupt` per target. It exits `1` when anything needs attention, so `testsprite agent status && …` can gate a CI step; `--dir ` inspects a different project root. + The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. @@ -220,6 +226,15 @@ testsprite test result test_xxxxxxxx --history --source cli --since 7d --output testsprite test result test_xxxxxxxx --history --dry-run --output json ``` +#### `testsprite test diff ` + +Compare two runs of a test and print what regressed: verdict, `failureKind`, `failedStepIndex`, per-step status flips, and `codeVersion` drift. Exit `0` when the verdicts match, `1` when they differ — so a script can assert "this rerun behaves like the last known-good run" in one call. + +```bash +testsprite test diff run_aaaa run_bbbb --output json +testsprite test diff run_aaaa run_bbbb --dry-run --output json +``` + #### `testsprite test failure get ` The latest-failure agent entry point. Returns one consistent snapshot of the latest failing run as a self-contained bundle: the result, the failed step plus its immediate neighbors with screenshots and DOM snapshots, the test source, a video pointer, a root-cause hypothesis, a recommended fix target, and correlation metadata. For the bundle of a _specific_ run an agent just triggered, prefer `test artifact get ` — it is keyed by `runId` and cannot be raced by another run that lands afterward. @@ -253,7 +268,28 @@ testsprite test failure summary test_xxxxxxxx --dry-run --output json ### Write commands -Require the `write:tests` scope. +Require the `write:tests` scope (project commands require `write:projects`), except `test scaffold` and `test lint`, which are pure-local authoring helpers — no network, no credentials, no scope. + +#### `testsprite test scaffold` + +Emit a schema-correct starter test definition — a frontend plan JSON by default, or a backend Python skeleton with `--type backend`. Pure-local: no network, no credentials. Edit the scaffold, then create the test with `--plan-from` / `--code-file`. + +```bash +testsprite test scaffold > first-test.plan.json +testsprite test scaffold --type backend --out tests/health.py +testsprite test scaffold --out plan.json --force # overwrite an existing file +``` + +#### `testsprite test lint` + +Validate plan/steps files offline with the same validators `test create` runs, collecting **every** problem instead of stopping at the first. No network, no credentials. Exit `0` when all inputs are valid, `5` otherwise. + +```bash +testsprite test lint --plan-from ./checkout.plan.json +testsprite test lint --plan-from-dir ./plans/ # every *.json checked, all errors reported +testsprite test lint --plans ./plans.jsonl # one plan spec per line +testsprite test lint --steps ./refined.plan.json # the shape `test plan put` ingests +``` #### `testsprite test create` @@ -292,7 +328,7 @@ testsprite test update test_xxxxxxxx --dry-run --output json #### `testsprite test delete ` / `test delete-batch` -Soft-delete one test (or many). `--confirm` is required; absent it, the CLI exits 5 with a local validation error. +Permanently delete one test (or many) — there is **no restore window**. `--confirm` is required; absent it, the CLI exits 5 with a local validation error. ```bash testsprite test delete test_xxxxxxxx --confirm @@ -322,20 +358,66 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Note the asymmetry: `--description` is **create-only** — `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`, but not `--description`. ```bash testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` +#### `testsprite project credential ` + +Set the **static backend credential** injected into every backend test in the project (free tier). Supported types: `public` (no credential), `"Bearer token"`, `"API key"`, `"basic token"`. + +```bash +testsprite project credential proj_xxxxxxxx --type "Bearer token" --credential-file ./token.txt +testsprite project credential proj_xxxxxxxx --type public +testsprite project credential proj_xxxxxxxx --type "API key" --credential sk-live-... --dry-run --output json +``` + +`--credential ` or `--credential-file ` supplies the value (required unless `--type public`). Prefer `--credential-file` in scripts so the secret never lands in shell history. + +#### `testsprite project auto-auth ` + +Configure the **recurring-token (auto-refresh) login** for backend tests (Pro): a fresh token is fetched on each run and injected into every backend test, so long-lived suites survive token expiry. + +```bash +# Password login: POST the login endpoint, extract the token, inject as a Bearer header +testsprite project auto-auth proj_xxxxxxxx \ + --method password --inject bearer \ + --login-url https://api.example.com/login --login-method POST \ + --login-content-type application/json \ + --login-body-template '{"user":"{{username}}","pass":"{{password}}"}' \ + --username ci@example.com --password-file ./pw.txt \ + --token-path '$.data.accessToken' + +# OAuth refresh-token flow +testsprite project auto-auth proj_xxxxxxxx \ + --method refresh_token --inject header --inject-key X-Auth-Token \ + --token-endpoint https://auth.example.com/oauth/token \ + --client-id my-client --client-secret-file ./secret.txt \ + --refresh-token-file ./refresh.txt --scope api.read + +# AWS Cognito refresh +testsprite project auto-auth proj_xxxxxxxx \ + --method aws_cognito_refresh --inject bearer \ + --client-id my-app-client --refresh-token-file ./refresh.txt --region us-east-1 + +# Turn it off (stored config is kept) +testsprite project auto-auth proj_xxxxxxxx --disable +``` + +Required flags: `--method ` and `--inject ` (`--inject-key ` names the header/cookie when not `bearer`). Method-specific flags: password login uses `--login-url/--login-method/--login-content-type/--login-body-template/--username/--password[-file]/--token-path`; OAuth uses `--token-endpoint/--client-id/--client-secret[-file]/--refresh-token[-file]/--scope`; Cognito adds `--region`. File variants (`--password-file`, `--client-secret-file`, `--refresh-token-file`) keep secrets out of shell history. + ### Run commands Require the `run:tests` scope. #### `testsprite test run ` -Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout` (with a `nextAction` pointing at `test wait ` so an agent can resume). +Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout`. On a timeout the CLI still prints the partial run object (with `runId`) to stdout **before** exiting 7, plus a `nextAction` pointing at `test wait ` — so a script always has the id to resume with, and stdout is never empty. + +`--all --project ` runs every test in the project in wave order. On the current unified engine that means **all tests, frontend and backend**; on the legacy backend-only engine, frontend tests can't run — they are skipped and enumerated in `skippedFrontend` with a stderr advisory. ```bash # Trigger and return immediately @@ -348,7 +430,7 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json -# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) +# Batch run with JUnit XML for CI (sidecar; --output json unchanged) testsprite test run --all --project proj_xxxxxxxx --wait \ --report junit --report-file ./results.xml --output json @@ -427,16 +509,17 @@ Flags: `--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. -#### `testsprite test wait ` +#### `testsprite test wait ` -Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. +Block until one **or more** runs reach a terminal status. With a single `run-id` the behavior is unchanged: same exit-code matrix as `test run --wait`. With several ids, the runs are polled concurrently under one shared `--timeout` and the CLI prints a `{ results, summary }` envelope — the worst status wins the exit code — so every re-attach hint the CLI prints can be pasted back as one command. `--max-concurrency ` (1–100, default 10) caps concurrent polls. Used to resume polling after a timed-out `--wait`, or when an agent already holds `runId`s from previous invocations. ```bash testsprite test wait run_01hx3z9p8q4k2y7a --timeout 600 --output json +testsprite test wait run_aaaa run_bbbb run_cccc --timeout 900 --output json testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json ``` -Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. #### `testsprite test artifact get ` @@ -451,6 +534,30 @@ testsprite test artifact get run_01hx3z9p8q4k2y7a --dry-run --output json Returns 404 (CLI exit 4) when the run passed (`details.reason: "no_failing_run"`), is still in flight (`run_not_ready`), was cancelled (`cancelled_no_artifacts`), or its test was deleted (`no_code`). +### Account & diagnostics + +#### `testsprite usage` (alias: `testsprite credits`) + +Account pre-flight before a large batch: resolves the active key to its identity (`userId`, `keyId`, `env`) and surfaces the credit balance / plan fields when the backend supplies them. Useful right before a `test run --all` fan-out. + +```bash +testsprite usage --output json +testsprite credits +testsprite usage --dry-run --output json +``` + +#### `testsprite doctor` + +One-shot environment diagnostic. Runs a fixed checklist — CLI version, Node.js runtime, active profile, API endpoint, credentials, live connectivity + key validity (`GET /me`), and whether the verify skill is installed in the current project — and prints an OK/WARN/FAIL report. Exits non-zero only when a check **fails** (warnings, e.g. skill not installed, don't fail the process), so it can gate a CI step or an agent preflight: + +```bash +testsprite doctor +testsprite doctor --output json +testsprite doctor && testsprite test run test_xxxxxxxx --wait +``` + +Every check reuses the same helpers the real commands use, so the report reflects exactly what a subsequent command would resolve. + ## Configuration ### Profiles & credentials @@ -473,14 +580,17 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ------------------------------- | --------------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | -| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | -| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| Variable | Purpose | +| ----------------------------------------- | ---------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support — API traffic is routed through the configured proxy | +| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | +| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | ### Update notice @@ -497,13 +607,14 @@ only outbound call the CLI makes besides your configured API endpoint. API-key scopes gate the write and run surfaces: -| Scope | Required by | -| --------------- | -------------------------------------------------------------------- | -| `read:me` | `auth whoami` | -| `read:projects` | `project list / get` | -| `read:tests` | every `test *` read command | -| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | -| `run:tests` | `test run / rerun / wait / artifact get` | +| Scope | Required by | +| ---------------- | -------------------------------------------------------------------- | +| `read:me` | `auth status`, `usage`, `doctor` (connectivity check) | +| `read:projects` | `project list / get` | +| `read:tests` | every `test *` read command | +| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | +| `write:projects` | `project create / update / credential / auto-auth` | +| `run:tests` | `test run / rerun / flaky / wait / artifact get` | New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. @@ -521,20 +632,25 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not ## Exit codes -| Code | Meaning | -| ---- | --------------------------------------- | -| `0` | Success | -| `1` | Generic failure / non-passed run status | -| `2` | Not yet implemented | -| `3` | Auth error | -| `4` | Not found | -| `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | -| `7` | Timeout / unsupported | -| `10` | Service unavailable | -| `11` | Rate limited (retriable) | -| `12` | Insufficient credits (non-retriable) | -| `13` | Feature gated (paid plan required) | +| Code | Meaning | +| --------------------- | --------------------------------------------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | +| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | + +### Signals & pipes + +On SIGINT (Ctrl-C), SIGTERM, or SIGHUP the CLI prints `Interrupted (). Any run already started keeps executing on the server; check it with 'testsprite test list' or 'testsprite test wait '.` and exits `128 + signal`. **Ctrl-C does not cancel the server-side run** — execution (and any credit spend) continues; there is no cancel command today, so re-attach with `test wait ` instead of re-triggering. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. ## Design principles diff --git a/README.md b/README.md index 404f237..78d293a 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude > **Pointing a coding agent (Claude Code, Cursor, Codex, Cline, …) at TestSprite?** Have it run `testsprite setup` first — that installs the verification skill, so the agent knows how to create, run, and triage tests on its own (instead of guessing from this README). New here? Start with the **[getting-started overview](https://docs.testsprite.com/cli/getting-started/overview)**. +> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). + From there, the loop runs on its own — an example session, typed by the coding agent: ```bash @@ -89,28 +91,34 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | +| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | +| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / permanently delete (no restore window; `--confirm` required) | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | +| | `test wait` | Block on one or more `runId`s until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/package.json b/package.json index 341e713..e78ce17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.2.0", + "version": "0.3.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", diff --git a/skills/testsprite-onboard.skill.md b/skills/testsprite-onboard.skill.md index cc9342c..df04c18 100644 --- a/skills/testsprite-onboard.skill.md +++ b/skills/testsprite-onboard.skill.md @@ -91,6 +91,27 @@ Each file is a COMPLETE plan and must include `projectId` (from step 2), `type: **Backend** — one `.py` file per endpoint, using `requests` with concrete assertions on status code and response body. +**Backend auth — read the injected `__AUTH_HEADERS__`, NEVER hardcode any credential.** This +covers **every** secret the API needs — Bearer/JWT tokens **and** API keys (`sk-…`, +`x-api-key`), basic-auth blobs, cookies. TestSprite prepends a managed credential block +(`__AUTH_CREDENTIAL__` / `__AUTH_TYPE__` / `__AUTH_HEADERS__`) to every backend test from the +project's Authentication settings, and `__AUTH_HEADERS__` already holds the right header(s) for +the configured type (Bearer → `{"Authorization": "Bearer …"}`; API key → `{"X-API-Key": "…"}`; +basic → `{"Authorization": "Basic …"}`). Spread it into your request headers — never paste a +literal `Bearer …` / `sk-…` / key value into the script: + +```python +r = requests.get(f"{TARGET_URL}/orders", headers={**__AUTH_HEADERS__}) +``` + +Configure the credential once on the project (ask the user for the value — never invent it or +reuse a key you happened to see): a static credential with +`testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential `, +or an auto-refreshing login with `testsprite project auto-auth …` so scheduled/repeat +runs keep working after the token expires. A hardcoded token expires within hours and a hardcoded +key can't be rotated centrally — `test create` emits a `[warn]` on an inlined credential; treat it +as a must-fix. + **Assertion rule (this is the whole game for FE):** every `assertion` step must name a **concrete, observable** outcome — an element, text, URL, count, or status. Never write `"verify it works"`, `"check the page loads"`, or other narrative that an AI judge can diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index 650a88b..fe7c661 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -165,6 +165,39 @@ only the Python **standard library + `requests` + `pytest` + `numpy` + `scipy`** - Get values from the API's responses (and captured variables), not by importing and calling the app's internals. +**Authentication — read the injected credential, NEVER hardcode any credential.** This +applies to **every** secret the API needs — Bearer/JWT tokens **and** API keys, basic-auth +blobs, session cookies. Do not paste a literal `Bearer …`, `sk-…`, `x-api-key` value, or any +other credential into the test. Before your script runs, TestSprite prepends a managed +credential block built from the project's Authentication settings, and `__AUTH_HEADERS__` +already contains the right header(s) for the configured auth type: + +```python +# Auto-injected credentials — do not modify +__AUTH_CREDENTIAL__ = "..." +__AUTH_TYPE__ = "Bearer token" # or "API key" / "basic token" / "public" +__AUTH_HEADERS__ = {"Authorization": "Bearer ..."} # API key → {"X-API-Key": "..."}; basic → {"Authorization": "Basic ..."} +``` + +Spread `__AUTH_HEADERS__` into every authenticated request — it adapts to whatever auth type +the project is configured for, so the same line works for Bearer, API-key, or basic auth: + +```python +r = requests.get(f"{TARGET_URL}/profile", headers={**__AUTH_HEADERS__}) +``` + +Configure the credential **once on the project** (ask the user for the value — never invent +or reuse a key you happened to see), and the block stays correct + refreshable: + +- **Bearer / API key / basic (static):** + `testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential ` +- **Auto-refreshing login (recurring token):** `testsprite project auto-auth …` + +A hardcoded token expires within hours (and a hardcoded key can't be rotated centrally), so +the test breaks on later runs; the managed block is rewritten with a fresh value each run. +`test create` emits a `[warn]` when it detects an inlined credential literal — treat that as +a must-fix, not a nuisance. + **Backend tests that share state declare dependencies at create time.** For a one-off verification, prefer a single self-contained script (log in inside the same file). But when the coverage set splits naturally into producer → consumer diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 4f5fdd3..24d8ec6 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -661,29 +661,39 @@ describe('runInstall — multi-target', () => { // --------------------------------------------------------------------------- describe('runInstall — empty target', () => { - it('non-TTY with no target throws exit 5', async () => { - const { fs: agentFs } = makeMemFs(); + it('non-TTY with no target defaults to claude and installs the skill file', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: [], + force: false, + }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); + + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + expect(store.has(claudeAbs)).toBe(true); + expect(capture.stderr.join('\n')).toContain('defaulting to claude'); + }); + + it('non-TTY default writes the canonical claude content', async () => { + const { store, fs: agentFs } = makeMemFs(); const { deps } = makeCapture(); - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: [], - force: false, - }, - { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, - ); - } catch (err) { - thrown = err; - } + await runInstall( + { profile: 'default', output: 'text', debug: false, dryRun: false, target: [], force: false }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); - expect(thrown).toBeInstanceOf(ApiError); - expect((thrown as ApiError).exitCode).toBe(5); + const { path: relPath, content } = renderForTarget('claude', 'testsprite-verify'); + const abs = path.resolve(CWD, relPath); + expect(store.get(abs)).toBe(content); }); it('TTY with injected prompt returning "claude" installs claude', async () => { diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 0a40c4b..7d7e7c8 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -371,18 +371,19 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (rawTargets.length === 0) { const isTTY = deps.isTTY ?? Boolean(process.stdin.isTTY); if (!isTTY) { - throw localValidationError( - 'target', - `required; pass --target=claude (comma-separated or repeated for several). Supported: ${Object.keys(TARGETS).join(', ')}`, + stderrFn( + '[info] --target not specified; defaulting to claude. Pass --target= to select a different agent.', ); + resolvedTargetStrings = ['claude']; + } else { + const promptFn = deps.prompt ?? ((q: string) => promptText(q)); + const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); + const defaulted = answer || 'claude'; + resolvedTargetStrings = defaulted + .split(',') + .map(s => s.trim()) + .filter(Boolean); } - const promptFn = deps.prompt ?? ((q: string) => promptText(q)); - const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); - const defaulted = answer || 'claude'; - resolvedTargetStrings = defaulted - .split(',') - .map(s => s.trim()) - .filter(Boolean); } else { resolvedTargetStrings = rawTargets; } diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 75fdff0..617fd6f 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -199,8 +199,14 @@ describe('runInit — happy path (interactive)', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('TestSprite initialized.'); expect(stdout).toContain('profile:'); + // Next steps leads with creating a project; no command that fails without --project. expect(stdout).toContain('Next steps:'); - expect(stdout).toContain('testsprite test list'); + expect(stdout).toContain('testsprite project create --type frontend'); + expect(stdout).toContain('testsprite test run --all --project '); + expect(stdout).toContain('the testsprite-onboard skill is installed'); + // No "current project" wording, no bare test list. + expect(stdout).not.toContain('current project'); + expect(stdout).not.toContain('testsprite test list'); }); it('json mode: emits structured InitSummary object', async () => { @@ -306,6 +312,15 @@ describe('runInit — --no-agent', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('skipped (--no-agent)'); + // --no-agent points at manual test creation; must not claim the skill is installed. + expect(stdout).toContain('Next steps:'); + expect(stdout).toContain('testsprite project create --type frontend'); + expect(stdout).toContain('testsprite test create --project '); + expect(stdout).toContain('testsprite test run --all --project '); + expect(stdout).not.toContain('skill is installed'); + // No "current project" wording, no bare test list. + expect(stdout).not.toContain('current project'); + expect(stdout).not.toContain('testsprite test list'); }); it('text mode with agent: summary contains skills line with both default skills', async () => { diff --git a/src/commands/init.ts b/src/commands/init.ts index aff0644..c2d5678 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -407,12 +407,33 @@ function renderInitText(data: unknown): string { } lines.push(''); lines.push('Next steps:'); - lines.push(' testsprite test list # list tests in the current project'); - lines.push(' testsprite agent list # check installed agent targets'); + lines.push(' # 1. Create your first project (frontend example) — prints a projectId'); + lines.push( + ' testsprite project create --type frontend --name "My App" --url https://your-app.com', + ); + lines.push(''); if (s.agent) { lines.push( - ' testsprite agent install --target= # re-install or install additional targets', + ' # 2. Generate tests: ask your coding agent (the testsprite-onboard skill is installed),', + ); + lines.push(' # or create one yourself, then run them (use the projectId from step 1):'); + lines.push(' testsprite test run --all --project '); + lines.push(''); + lines.push(' # Manage installed agent skills'); + lines.push(' testsprite agent list'); + lines.push( + ' testsprite agent install --target= # re-install or install additional targets', + ); + } else { + lines.push(' # 2. Create a test, then run it (use the projectId from step 1):'); + lines.push(' testsprite test create --project ...'); + lines.push(' testsprite test run --all --project '); + lines.push( + ' # Tip: `testsprite agent install` sets up the onboarding skill for your coding agent', ); + lines.push(''); + lines.push(' # Manage installed agent skills'); + lines.push(' testsprite agent list'); } return lines.join('\n'); diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index f63928c..daea851 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -8,7 +8,9 @@ import { type CliProject, type CliUpdateProjectResponse, createProjectCommand, + runAutoAuth, runCreate, + runCredential, runGet, runList, runUpdate, @@ -72,10 +74,10 @@ describe('createProjectCommand', () => { errorSpy.mockRestore(); }); - it('exposes list, get, create and update subcommands', () => { + it('exposes list, get, create, update, credential and auto-auth subcommands', () => { const project = createProjectCommand(); const names = project.commands.map(c => c.name()).sort(); - expect(names).toEqual(['create', 'get', 'list', 'update']); + expect(names).toEqual(['auto-auth', 'create', 'credential', 'get', 'list', 'update']); }); it('list exposes the pagination flags from the design contract', () => { @@ -327,6 +329,21 @@ describe('runList', () => { }); }); +describe('DEV-244 — project update no longer accepts the dead --description flag', () => { + it('rejects --description on `project update` as an unknown option', async () => { + const project = createProjectCommand(); + const update = project.commands.find(c => c.name() === 'update')!; + project.exitOverride(); + update.exitOverride(); + + await expect( + project.parseAsync(['update', 'proj_x', '--description', 'should not exist'], { + from: 'user', + }), + ).rejects.toThrow(/unknown option.*--description/i); + }); +}); + describe('createProjectCommand --page-size option parser', () => { it('rejects non-numeric --page-size values via commander', async () => { const project = createProjectCommand(); @@ -864,7 +881,6 @@ describe('runUpdate', () => { debug: false, projectId: 'proj_text', name: 'New Name', - description: 'New desc', }, { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, ); @@ -929,3 +945,249 @@ describe('runUpdate', () => { expect(result.updatedFields).toBeUndefined(); }); }); + +describe('runCredential', () => { + interface Captured { + url: string; + method: string; + body: unknown; + headers: Headers; + } + function captureFetch(captured: Captured[], body: unknown) { + return makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { status: 200, body }; + }); + } + + it('PUTs /projects/:id/credential with authType + credential + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured, { + projectId: 'p1', + authType: 'Bearer token', + rewroteCount: 2, + }); + const res = await runCredential( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + authType: 'Bearer token', + credential: 'tok-123', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(res.rewroteCount).toBe(2); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.url).toContain('/projects/p1/credential'); + expect(put.body).toEqual({ authType: 'Bearer token', credential: 'tok-123' }); + expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-cred-[0-9a-f-]{36}$/); + }); + + it('public clears the credential (no credential in body, none required)', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured, { + projectId: 'p1', + authType: 'public', + rewroteCount: 0, + }); + await runCredential( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'public' }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.body).toEqual({ authType: 'public' }); + }); + + it('non-public without --credential → VALIDATION_ERROR (exit 5), no fetch', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runCredential( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'API key' }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); + + it('rejects an unknown --type locally (no fetch)', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runCredential( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + authType: 'jwt', + credential: 'x', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('runAutoAuth', () => { + interface Captured { + url: string; + method: string; + body: Record; + headers: Headers; + } + function captureFetch(captured: Captured[]) { + return makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : {}, + headers: new Headers(init.headers as Record), + }); + return { + status: 200, + body: { projectId: 'p1', enabled: true, method: 'aws_cognito_refresh', inject: 'bearer' }, + }; + }); + } + + it('PUTs /projects/:id/auto-auth with the config body + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'aws_cognito_refresh', + inject: 'bearer', + region: 'us-east-1', + clientId: 'abc', + refreshToken: 'rt-xyz', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.url).toContain('/projects/p1/auto-auth'); + expect(put.body).toEqual({ + enabled: true, + method: 'aws_cognito_refresh', + inject: 'bearer', + region: 'us-east-1', + clientId: 'abc', + refreshToken: 'rt-xyz', + }); + expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-autoauth-[0-9a-f-]{36}$/); + }); + + it('--disable sends enabled:false', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + disable: true, + method: 'password', + inject: 'bearer', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(captured.find(c => c.method === 'PUT')!.body.enabled).toBe(false); + }); + + it('reads a secret from --refresh-token-file', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-rt-')); + const rtFile = join(dir, 'rt.txt'); + writeFileSync(rtFile, ' rt-from-file\n'); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'refresh_token', + inject: 'bearer', + tokenEndpoint: 'https://idp.example.com/token', + refreshTokenFile: rtFile, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(captured.find(c => c.method === 'PUT')!.body.refreshToken).toBe('rt-from-file'); + }); + + it('rejects an unknown --method / --inject locally (no fetch)', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'magic', + inject: 'bearer', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('dogfood 2026-06-30 — whitespace-only --name is rejected (parity with `test create`)', () => { + const noNetwork = () => { + throw new Error('network should not be hit'); + }; + + it('runCreate rejects a whitespace-only --name (exit 5, no network)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runCreate( + { profile: 'default', output: 'json', debug: false, type: 'backend', name: ' ' }, + { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('runUpdate rejects a whitespace-only --name (exit 5, no network)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runUpdate( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', name: '\t \n' }, + { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); diff --git a/src/commands/project.ts b/src/commands/project.ts index ef4d5e8..96f2fc8 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -142,20 +142,16 @@ export async function runCreate( // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. assertIdempotencyKey(opts.idempotencyKey); - // Reject empty / whitespace-only names so a junk record never reaches the - // backend — matches the `requireString` whitespace guard `test create` uses - // (dogfood P1 fix #1). Without this, `--name " "` passes the action - // handler's `if (!name)` check (a non-empty string is truthy) and is sent - // verbatim, creating a blank-named project. - if (opts.name !== undefined && opts.name.trim().length === 0) { - throw localValidationError('--name must not be empty or whitespace-only'); + // P1-3: client-side length checks matching server limits. + // Whitespace-only / empty rejection (parity with `test create`'s requireString; + // a truthy `--name " "` otherwise creates a blank-named project on the backend). + if (opts.name === undefined || opts.name.trim().length === 0) { + throw localValidationError('--name is required and must not be empty or whitespace-only'); } if (opts.password !== undefined && opts.password.trim().length === 0) { throw localValidationError('--password must not be empty or whitespace-only'); } - - // P1-3: client-side length checks matching server limits. - if (opts.name !== undefined && opts.name.length > 200) { + if (opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } if (opts.description !== undefined && opts.description.length > 2000) { @@ -248,7 +244,6 @@ interface UpdateOptions extends CommonOptions { username?: string; password?: string; passwordFile?: string; - description?: string; instruction?: string; idempotencyKey?: string; } @@ -264,6 +259,8 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. + // Reject a whitespace-only `--name` on update too (parity with create); name + // stays optional here, so only validate when the flag is supplied. if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } @@ -273,10 +270,6 @@ export async function runUpdate( if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } - if (opts.description !== undefined && opts.description.length > 2000) { - throw localValidationError('--description must be at most 2000 characters'); - } - // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); @@ -288,7 +281,6 @@ export async function runUpdate( targetUrl: opts.targetUrl !== undefined, username: opts.username !== undefined, password: passwordSupplied, - description: opts.description !== undefined, instruction: opts.instruction !== undefined, }; const presentFieldNames = Object.entries(mutableFields) @@ -296,7 +288,7 @@ export async function runUpdate( .map(([field]) => field); if (presentFieldNames.length === 0) { throw localValidationError( - 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', + 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, or --instruction.', ); } @@ -336,7 +328,6 @@ export async function runUpdate( targetUrl: opts.targetUrl, username: opts.username, password, - description: opts.description, instruction: opts.instruction, }; const body = Object.fromEntries( @@ -355,6 +346,228 @@ export async function runUpdate( return updated; } +// --------------------------------------------------------------------------- +// project credential — set the static backend credential +// --------------------------------------------------------------------------- + +const CLI_AUTH_TYPES = ['public', 'Bearer token', 'API key', 'basic token'] as const; + +export interface CliProjectCredentialResponse { + projectId: string; + authType: string; + rewroteCount: number; +} + +interface CredentialOptions extends CommonOptions { + projectId: string; + authType: string; + credential?: string; + credentialFile?: string; + idempotencyKey?: string; +} + +export async function runCredential( + opts: CredentialOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + assertIdempotencyKey(opts.idempotencyKey); + + if (!(CLI_AUTH_TYPES as readonly string[]).includes(opts.authType)) { + throw localValidationError(`--type must be one of: ${CLI_AUTH_TYPES.join(', ')}`); + } + + // Resolve the credential value (flag or file). Required for every type + // except `public` (which clears it). + let credential = opts.credential; + if (credential === undefined && opts.credentialFile !== undefined) { + credential = readFileSync(opts.credentialFile, 'utf8').trim(); + } + if (opts.authType !== 'public' && (credential === undefined || credential === '')) { + throw localValidationError( + '--credential (or --credential-file) is required unless --type is "public"', + ); + } + + const body: Record = { authType: opts.authType }; + if (opts.authType !== 'public' && credential !== undefined) body.credential = credential; + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-cred-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + if (opts.dryRun) { + const sample: CliProjectCredentialResponse = { + projectId: opts.projectId, + authType: opts.authType, + rewroteCount: 0, + }; + out.print(sample, data => renderCredentialText(data as CliProjectCredentialResponse)); + return sample; + } + + const client = makeClient(opts, deps); + const res = await client.put( + `/projects/${encodeURIComponent(opts.projectId)}/credential`, + { body, headers: { 'idempotency-key': idempotencyKey } }, + ); + out.print(res, data => renderCredentialText(data as CliProjectCredentialResponse)); + return res; +} + +function renderCredentialText(r: CliProjectCredentialResponse): string { + return [ + `projectId ${r.projectId}`, + `authType ${r.authType}`, + `rewroteCount ${r.rewroteCount}`, + ].join('\n'); +} + +// --------------------------------------------------------------------------- +// project auto-auth — configure the recurring-token (auto-refresh) login +// --------------------------------------------------------------------------- + +const AUTO_AUTH_METHODS = ['password', 'refresh_token', 'aws_cognito_refresh'] as const; +const AUTO_AUTH_INJECTS = ['bearer', 'header', 'cookie'] as const; + +export interface CliProjectAutoAuthResponse { + projectId: string; + enabled: boolean; + method: string; + inject: string; + /** + * Present when the server's trial refresh failed: `enabled` is then `false` + * and this carries the reason (e.g. a bad refresh token). The config is still + * stored, but auto-auth won't run until the login succeeds. + */ + lastRefreshError?: string; +} + +interface AutoAuthOptions extends CommonOptions { + projectId: string; + disable?: boolean; + method: string; + inject: string; + injectKey?: string; + // password method + loginUrl?: string; + loginMethod?: string; + loginContentType?: string; + loginBodyTemplate?: string; + username?: string; + password?: string; + passwordFile?: string; + tokenPath?: string; + // refresh_token method + tokenEndpoint?: string; + clientId?: string; + clientSecret?: string; + clientSecretFile?: string; + refreshToken?: string; + refreshTokenFile?: string; + scope?: string; + // aws_cognito_refresh method + region?: string; + idempotencyKey?: string; +} + +export async function runAutoAuth( + opts: AutoAuthOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + assertIdempotencyKey(opts.idempotencyKey); + + if (!(AUTO_AUTH_METHODS as readonly string[]).includes(opts.method)) { + throw localValidationError(`--method must be one of: ${AUTO_AUTH_METHODS.join(', ')}`); + } + if (!(AUTO_AUTH_INJECTS as readonly string[]).includes(opts.inject)) { + throw localValidationError(`--inject must be one of: ${AUTO_AUTH_INJECTS.join(', ')}`); + } + + // Resolve secrets from --*-file variants so they stay out of shell history. + const password = + opts.password ?? + (opts.passwordFile !== undefined ? readFileSync(opts.passwordFile, 'utf8').trim() : undefined); + const clientSecret = + opts.clientSecret ?? + (opts.clientSecretFile !== undefined + ? readFileSync(opts.clientSecretFile, 'utf8').trim() + : undefined); + const refreshToken = + opts.refreshToken ?? + (opts.refreshTokenFile !== undefined + ? readFileSync(opts.refreshTokenFile, 'utf8').trim() + : undefined); + + const enabled = opts.disable !== true; + const body: Record = { enabled, method: opts.method, inject: opts.inject }; + const maybe = (k: string, v: string | undefined): void => { + if (v !== undefined) body[k] = v; + }; + maybe('injectKey', opts.injectKey); + maybe('loginUrl', opts.loginUrl); + maybe('loginMethod', opts.loginMethod); + maybe('loginContentType', opts.loginContentType); + maybe('loginBodyTemplate', opts.loginBodyTemplate); + maybe('username', opts.username); + maybe('password', password); + maybe('tokenPath', opts.tokenPath); + maybe('tokenEndpoint', opts.tokenEndpoint); + maybe('clientId', opts.clientId); + maybe('clientSecret', clientSecret); + maybe('refreshToken', refreshToken); + maybe('scope', opts.scope); + maybe('region', opts.region); + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-autoauth-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + if (opts.dryRun) { + const sample: CliProjectAutoAuthResponse = { + projectId: opts.projectId, + enabled, + method: opts.method, + inject: opts.inject, + }; + out.print(sample, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); + return sample; + } + + const client = makeClient(opts, deps); + const res = await client.put( + `/projects/${encodeURIComponent(opts.projectId)}/auto-auth`, + { body, headers: { 'idempotency-key': idempotencyKey } }, + ); + out.print(res, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); + return res; +} + +function renderAutoAuthText(r: CliProjectAutoAuthResponse): string { + const lines = [ + `projectId ${r.projectId}`, + `enabled ${r.enabled}`, + `method ${r.method}`, + `inject ${r.inject}`, + ]; + if (r.lastRefreshError) { + lines.push(`lastRefreshError ${r.lastRefreshError}`); + } + // A disabled result after a write means the trial login failed — call it out + // so the user doesn't assume auto-auth is live. + if (!r.enabled) { + lines.push( + 'note auto-auth was stored but is DISABLED — the trial login failed. Fix the credentials (e.g. a valid refresh token) and re-run.', + ); + } + return lines.join('\n'); +} + export function createProjectCommand(deps: ProjectDeps = {}): Command { const project = new Command('project').description('Manage TestSprite projects'); @@ -448,7 +661,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--username ', 'new auth username') .option('--password ', 'new auth password') .option('--password-file ', 'read new password from file') - .option('--description ', 'new description') .option('--instruction ', 'new FE plan-gen instruction hint') .option( '--idempotency-key ', @@ -465,7 +677,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { username: cmdOpts.username, password: cmdOpts.password, passwordFile: cmdOpts.passwordFile, - description: cmdOpts.description, instruction: cmdOpts.instruction, idempotencyKey: cmdOpts.idempotencyKey, }, @@ -473,6 +684,99 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { ); }); + project + .command('credential ') + .description( + 'Set the static backend credential injected into every backend test\n' + + '(Bearer token / API key / Basic token / public). Free tier.', + ) + .requiredOption('--type ', 'public | "Bearer token" | "API key" | "basic token"') + .option('--credential ', 'credential value (required unless --type public)') + .option('--credential-file ', 'read the credential value from a file') + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: CredentialFlagOpts, command: Command) => { + await runCredential( + { + ...resolveCommonOptions(command), + projectId, + authType: cmdOpts.type, + credential: cmdOpts.credential, + credentialFile: cmdOpts.credentialFile, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + project + .command('auto-auth ') + .description( + 'Configure the recurring-token (auto-refresh login) for backend tests (Pro).\n' + + 'A fresh token is fetched on each run and injected into every backend test.', + ) + .requiredOption('--method ', 'password | refresh_token | aws_cognito_refresh') + .requiredOption('--inject ', 'bearer | header | cookie') + .option('--disable', 'turn auto-auth off (keeps stored config)') + .option('--inject-key ', 'header/cookie name when --inject is header/cookie') + // password method + .option('--login-url ', 'login endpoint (method=password)') + .option('--login-method ', 'POST | PUT (method=password)') + .option('--login-content-type ', 'application/json | application/x-www-form-urlencoded') + .option('--login-body-template ', 'login body template with {{username}}/{{password}}') + .option('--username ', 'login username (method=password)') + .option('--password ', 'login password (method=password)') + .option('--password-file ', 'read login password from a file') + .option('--token-path ', 'JSONPath to the token in the login response') + // refresh_token method + .option('--token-endpoint ', 'OAuth token endpoint (method=refresh_token)') + .option('--client-id ', 'OAuth client id') + .option('--client-secret ', 'OAuth client secret') + .option('--client-secret-file ', 'read OAuth client secret from a file') + .option('--refresh-token ', 'OAuth/Cognito refresh token') + .option('--refresh-token-file ', 'read the refresh token from a file') + .option('--scope ', 'OAuth scope') + // aws_cognito_refresh method + .option('--region ', "AWS region (method=aws_cognito_refresh, e.g. 'us-east-1')") + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: AutoAuthFlagOpts, command: Command) => { + await runAutoAuth( + { + ...resolveCommonOptions(command), + projectId, + disable: cmdOpts.disable, + method: cmdOpts.method, + inject: cmdOpts.inject, + injectKey: cmdOpts.injectKey, + loginUrl: cmdOpts.loginUrl, + loginMethod: cmdOpts.loginMethod, + loginContentType: cmdOpts.loginContentType, + loginBodyTemplate: cmdOpts.loginBodyTemplate, + username: cmdOpts.username, + password: cmdOpts.password, + passwordFile: cmdOpts.passwordFile, + tokenPath: cmdOpts.tokenPath, + tokenEndpoint: cmdOpts.tokenEndpoint, + clientId: cmdOpts.clientId, + clientSecret: cmdOpts.clientSecret, + clientSecretFile: cmdOpts.clientSecretFile, + refreshToken: cmdOpts.refreshToken, + refreshTokenFile: cmdOpts.refreshTokenFile, + scope: cmdOpts.scope, + region: cmdOpts.region, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + return project; } @@ -500,11 +804,41 @@ interface UpdateFlagOpts { username?: string; password?: string; passwordFile?: string; - description?: string; instruction?: string; idempotencyKey?: string; } +interface CredentialFlagOpts { + type: string; + credential?: string; + credentialFile?: string; + idempotencyKey?: string; +} + +interface AutoAuthFlagOpts { + disable?: boolean; + method: string; + inject: string; + injectKey?: string; + loginUrl?: string; + loginMethod?: string; + loginContentType?: string; + loginBodyTemplate?: string; + username?: string; + password?: string; + passwordFile?: string; + tokenPath?: string; + tokenEndpoint?: string; + clientId?: string; + clientSecret?: string; + clientSecretFile?: string; + refreshToken?: string; + refreshTokenFile?: string; + scope?: string; + region?: string; + idempotencyKey?: string; +} + function parseFlag(raw: string | undefined, flagName: string): number | undefined { if (raw === undefined) return undefined; const n = Number(raw); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 6ae831c..f7c7602 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -4631,6 +4631,48 @@ describe('runCreate', () => { expect(sent.headers.get('x-api-key')).toBe('sk-user-test'); }); + it('emits backend warnings[] to stderr without polluting stdout JSON', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('BEARER = "eyJhbGciOi.eyJzdWIiOiJ4In0.sig"\n'); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + ...SAMPLE_RESPONSE, + type: 'backend', + warnings: [ + 'This test appears to hardcode an auth credential — read auth from __AUTH_HEADERS__.', + ], + }, + }; + }); + const out: string[] = []; + const err: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'backend', + name: 'hardcoded be', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => err.push(line), + }, + ); + // Warning lands on stderr, prefixed `[warn]`. + expect(err.some(l => l.includes('[warn]') && l.includes('__AUTH_HEADERS__'))).toBe(true); + // stdout stays the parseable wire object — no warning noise. + expect(out.join('\n')).not.toContain('[warn]'); + }); + it('respects a caller-supplied --idempotency-key (for safe retries)', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('code body'); @@ -5674,6 +5716,36 @@ describe('runCreate — M4 BE dependency authoring flags', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('[FE guard] --produces rejection is well-formed: bare field, no ----produces, singular verb (dogfood 2026-06-30)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// fe code'); + const err = (await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_fe', + type: 'frontend', + name: 'fe test', + codeFile, + produces: ['some_var'], + }, + { + credentialsPath, + fetchImpl: () => Promise.resolve(new Response('{}')), + stdout: () => undefined, + stderr: () => undefined, + }, + ).catch((e: unknown) => e)) as ApiError; + expect(err).toBeInstanceOf(ApiError); + // details.field is the BARE flag name (was '--produces' → double-dashed subject). + expect(err.details).toMatchObject({ field: 'produces' }); + expect(err.nextAction).toContain('--produces'); + expect(err.nextAction).not.toContain('----'); // regression: '----produces' + expect(err.nextAction).toContain('is a backend-only flag'); // singular for one flag + expect(err.nextAction).not.toContain('backend..'); // regression: double period + }); + it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --needs', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('// fe code'); diff --git a/src/commands/test.ts b/src/commands/test.ts index 1c6edd8..0602afc 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -543,6 +543,12 @@ export interface CliCreateTestResponse { type: 'frontend' | 'backend'; codeVersion: string; createdAt: string; + /** + * Non-fatal advisories from the backend (e.g. the BE auth guardrail + * flagging a hardcoded credential). Rendered on stderr; the create + * still succeeded. + */ + warnings?: string[]; } export const CLI_CREATE_PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; @@ -752,14 +758,19 @@ export async function runCreate( // save a round-trip. if (opts.type === 'frontend') { const depFlags: string[] = []; - if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('--produces'); - if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('--needs'); - if (opts.category !== undefined) depFlags.push('--category'); + if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('produces'); + if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('needs'); + if (opts.category !== undefined) depFlags.push('category'); if (depFlags.length > 0) { + // Pass the BARE flag name to localValidationError — its kind:'flag' branch + // adds the `--` prefix, so '--produces' would render as '----produces'. + const flagList = depFlags.map(f => `--${f}`); + const verb = depFlags.length === 1 ? 'is a backend-only flag' : 'are backend-only flags'; + // No trailing period: localValidationError appends one after the reason. throw localValidationError( depFlags[0]!, - `${depFlags.join(', ')} are backend-only flags; frontend plans have no wave model. ` + - `Remove ${depFlags.join('/')} or use --type backend.`, + `${flagList.join(', ')} ${verb}; frontend plans have no wave model. ` + + `Remove ${flagList.join('/')} or use --type backend`, ); } } @@ -836,6 +847,11 @@ export async function runCreate( headers: { 'idempotency-key': idempotencyKey }, }); + // Surface backend advisories (e.g. a hardcoded-credential warning for BE + // tests) on stderr so they reach the agent without polluting stdout JSON. + // Emitted before the --run early return so they always show. + emitResponseWarnings(response.warnings, deps); + // --run chain (M3.3 piece-3). Per codex round-1 P1: suppress the // create's own print when chaining; `runTestRun` emits a single // merged envelope `{ ...createResponse, run: }` so @@ -998,6 +1014,16 @@ function renderCreateText(response: CliCreateTestResponse): string { ].join('\n'); } +/** + * Emit backend `warnings[]` advisories to stderr (one `[warn]` line each), + * keeping stdout — JSON or text — uncluttered. No-op when absent/empty. + */ +function emitResponseWarnings(warnings: string[] | undefined, deps: TestDeps): void { + if (!warnings || warnings.length === 0) return; + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + for (const w of warnings) stderrFn(`[warn] ${w}`); +} + /** * §6.X / M3.2 piece-6 — response from `PUT /tests/{id}/plan-steps`. * `planStepsHash` is a sha256 over the canonicalized new array so @@ -3400,6 +3426,12 @@ export interface CliPutTestCodeResponse { testId: string; codeVersion: string; updatedAt: string; + /** + * Non-fatal advisories (e.g. the BE auth guardrail flagging a hardcoded + * credential in the replaced code). Rendered on stderr; the update still + * succeeded. + */ + warnings?: string[]; } type CodePutLanguage = CliTestCode['language']; @@ -3577,6 +3609,7 @@ export async function runCodePut( }, }, ); + emitResponseWarnings(response.warnings, deps); out.print(response, data => renderCodePutText(data as CliPutTestCodeResponse)); return response; } catch (err) { @@ -5803,7 +5836,7 @@ export async function runTestWait( // --------------------------------------------------------------------------- interface RunTestRunAllOptions extends CommonOptions { - /** projectId to run all BE tests in. */ + /** projectId to run all tests in. */ projectId: string; /** --filter : only run tests whose name contains this substring (case-insensitive). */ nameFilter?: string; @@ -5940,7 +5973,7 @@ export async function runTestRunAll( stderrFn(`idempotency-key: ${idempotencyKey}`); } - // Resolve testIds: fetch all BE tests in the project, apply --filter. + // Resolve testIds: fetch all tests in the project, apply --filter. let testIds: string[] | undefined; if (opts.nameFilter !== undefined && opts.nameFilter !== '') { // We need to resolve the full test set to apply the name filter. @@ -5978,7 +6011,8 @@ export async function runTestRunAll( `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch run.`, ); } - // When no --filter, omit testIds → server runs ALL BE tests in the project. + // When no --filter, omit testIds → server runs ALL tests in the project + // (BE tests on the legacy V2 wave engine; FE + BE on the V3 unified engine). const batchResp = await client.triggerBatchRunFresh( { @@ -8369,7 +8403,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { .command('run [test-id]') .description( 'Trigger a test run. With --wait, polls until terminal status.\n' + - 'Use --all --project for a wave-ordered batch run of all BE tests (M4).\n' + + 'Use --all --project for a wave-ordered batch run of all tests in a project (M4).\n' + '\nExit codes:\n' + ' 0 passed (or queued without --wait)\n' + ' 1 failed / blocked / cancelled\n' + @@ -8397,7 +8431,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--all', - 'run all BE tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', + 'run all tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', false, ) .option( @@ -8424,11 +8458,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + - ' testsprite test run --all --project run all BE tests in wave order\n' + + ' testsprite test run --all --project run all project tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + - '(see `testsprite test create --help` for details).', + '(see `testsprite test create --help` for details).\n' + + '\nFrontend tests: the current unified engine runs FE tests too (they are billed\n' + + 'like any run). On the legacy backend-only engine FE tests cannot run — they are\n' + + "reported under skippedFrontend with an advisory; run those with 'test run '.", ) .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testIdArg: string | undefined, cmdOpts: RunFlagOpts, command: Command) => { @@ -8444,7 +8481,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (testIdArg === undefined && !isAll) { throw localValidationError( 'test-id', - 'provide a , or use --all --project to run all BE tests in a project', + 'provide a , or use --all --project to run all tests in a project', ); } // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). @@ -8473,14 +8510,16 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--all requires a project id — pass --project ', ); } - // --target-url has no effect on the --all batch path: it is BE-only - // (FE tests are skipped server-side) and a BE test's base URL is baked - // into its code. Silently dropping it could run the suite against an - // unintended environment in the caller's mind — reject loudly instead. + // --target-url has no effect on the --all batch path: a BE test's base + // URL is baked into its code, and the unified engine resolves each + // project's configured environment server-side (per-run URL overrides + // are not applied to batch FE runs either). Silently dropping it could + // run the suite against an unintended environment in the caller's mind + // — reject loudly instead. if (cmdOpts.targetUrl !== undefined && cmdOpts.targetUrl !== '') { throw localValidationError( 'target-url', - '--target-url has no effect with --all (the batch path is the BE-only wave engine; a BE test’s URL is baked into its code). Remove --target-url.', + '--target-url has no effect with --all (the batch path does not apply a per-run URL override — BE test URLs are baked into their code and the unified engine resolves the project environment server-side). Remove --target-url.', ); } await runTestRunAll( @@ -9973,7 +10012,11 @@ export function createTestArtifactCommand(deps: TestDeps): Command { 'Parent must exist. The bundle dir itself is created if absent.', ].join(' '), ) - .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .option( + '--failed-only', + 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + + 'so this is usually a no-op; use `test steps ` for the full run trail.', + ) .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (runId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { @@ -10003,7 +10046,11 @@ function createTestFailureCommand(deps: TestDeps): Command { '--out ', 'Directory to write the §7 disk layout into (default: print wire envelope to stdout)', ) - .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .option( + '--failed-only', + 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + + 'so this is usually a no-op; use `test steps ` for the full run trail.', + ) .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (testId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { diff --git a/src/commands/usage.ts b/src/commands/usage.ts index a97dbba..1980d26 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -154,8 +154,12 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { } if (u.creditsPerRun !== undefined) { lines.push(`cost per frontend run: ${u.creditsPerRun} credit(s)`); + // Backend runs DO consume credits (confirmed by design 2026-06-30 / DEV-289). + // The API exposes no backend-specific per-run cost field, and it differs from + // the frontend rate, so state that it bills without asserting a possibly-wrong + // number — check your balance before/after, or see the billing page. lines.push( - `cost per backend run: 0 credit(s) (backend tests bill at code-generation, not at run time)`, + `cost per backend run: also consumes credits (exact amount not reported by the API)`, ); } diff --git a/src/index.ts b/src/index.ts index 806f6e4..bf9a9aa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,7 +52,7 @@ program .option('--debug', 'Print HTTP method/path, request id, latency, retry decisions to stderr') .option( '--dry-run', - 'Skip the network, credentials, and filesystem; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key.', + 'Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. Note: file inputs you pass (--plan-from/--plans/--steps) are still read and validated locally; only --code-file uses a placeholder.', ) .option( '--request-timeout ', diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 51bf9b5..2ac9158 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -34,7 +34,9 @@ import { * The description value is a single line (no folded/literal block scalars). */ function parseFrontmatterDescription(content: string): string | undefined { - const lines = content.split('\n'); + // Tolerate CRLF so a Windows checkout (autocrlf) doesn't leave a trailing + // \r on the description and break the byte-identical comparisons. + const lines = content.split(/\r?\n/); let inFrontmatter = false; for (const line of lines) { if (line.trim() === '---') { diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index aec451a..ff20e8c 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -10,7 +10,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { isAbsolute, join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; import { applyFailedOnly, @@ -593,8 +593,8 @@ describe('resolveBundleDir', () => { it('resolves a relative path against cwd', () => { const out = resolveBundleDir('./tmp/x'); - expect(out.endsWith('/tmp/x')).toBe(true); - expect(out.startsWith('/')).toBe(true); + expect(out).toBe(resolve(process.cwd(), 'tmp', 'x')); + expect(isAbsolute(out)).toBe(true); }); it('strips a trailing slash', () => { diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index 896d057..d50ad52 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -1,5 +1,5 @@ import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { @@ -139,8 +139,11 @@ describe('writeProfile', () => { it('creates the file with mode 0600 and writes the profile', () => { writeProfile(DEFAULT_PROFILE, { apiKey: 'sk-new' }, { path: credentialsPath }); expect(existsSync(credentialsPath)).toBe(true); - const mode = statSync(credentialsPath).mode & 0o777; - expect(mode).toBe(0o600); + // POSIX file modes don't exist on Windows (stat reports 0666). + if (process.platform !== 'win32') { + const mode = statSync(credentialsPath).mode & 0o777; + expect(mode).toBe(0o600); + } expect(readProfile(DEFAULT_PROFILE, { path: credentialsPath })).toEqual({ apiKey: 'sk-new' }); }); @@ -188,7 +191,8 @@ describe('ensureRestrictiveMode', () => { expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); }); - it('downgrades over-permissive modes', () => { + // POSIX-only premise: Windows has no 0644/0600 distinction to downgrade. + it.skipIf(process.platform === 'win32')('downgrades over-permissive modes', () => { mkdirSync(tmpRoot, { recursive: true }); writeFileSync(credentialsPath, 'data', { mode: 0o644 }); ensureRestrictiveMode(credentialsPath); @@ -199,7 +203,7 @@ describe('ensureRestrictiveMode', () => { describe('defaultCredentialsPath', () => { it('points at ~/.testsprite/credentials', () => { - expect(defaultCredentialsPath().endsWith('/.testsprite/credentials')).toBe(true); + expect(defaultCredentialsPath()).toBe(join(homedir(), '.testsprite', 'credentials')); }); }); diff --git a/src/lib/http.ts b/src/lib/http.ts index af1c25a..7a9051a 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -528,9 +528,9 @@ export class HttpClient { } // Edge proxies / load balancers return 408/502/504 without our error - // envelope on transient outages. Per the CLI error spec §7 these are - // transport-level retries, not facade errors — fold them in here so - // we get the bounded backoff budget instead of a single INTERNAL bail. + // envelope on transient outages. These are transport-level retries, + // not facade errors — fold them in here so we get the bounded backoff + // budget instead of a single INTERNAL bail. if (rawBody === null && isTransportEdgeStatus(response.status)) { this.debug({ kind: 'error', diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index c15b09d..2b26c0e 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -13,24 +13,30 @@ import { // isVerifySkillInstalled // --------------------------------------------------------------------------- +// The implementation joins paths with the native separator; normalize so the +// fakes below match on Windows (backslashes) as well as POSIX. +const toPosix = (p: string) => p.replaceAll('\\', '/'); + describe('isVerifySkillInstalled', () => { it('true when the claude own-file SKILL.md exists', () => { - const existsSync = (p: string) => p.endsWith('.claude/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => + toPosix(p).endsWith('.claude/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cursor .mdc landing file', () => { - const existsSync = (p: string) => p.endsWith('.cursor/rules/testsprite-verify.mdc'); + const existsSync = (p: string) => toPosix(p).endsWith('.cursor/rules/testsprite-verify.mdc'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cline landing file', () => { - const existsSync = (p: string) => p.endsWith('.clinerules/testsprite-verify.md'); + const existsSync = (p: string) => toPosix(p).endsWith('.clinerules/testsprite-verify.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the antigravity landing file', () => { - const existsSync = (p: string) => p.endsWith('.agents/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => + toPosix(p).endsWith('.agents/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); @@ -73,7 +79,7 @@ describe('isVerifySkillInstalled', () => { return false; }, }); - expect(seen.every(p => p.startsWith('/some/proj'))).toBe(true); + expect(seen.every(p => toPosix(p).startsWith('/some/proj'))).toBe(true); // One probe per target landing path. expect(seen).toHaveLength(Object.keys(TARGETS).length); }); @@ -198,6 +204,6 @@ describe('maybeEmitSkillNudge', () => { }); maybeEmitSkillNudge(ctx); expect(probed.length).toBeGreaterThan(0); - expect(probed.every(p => p.startsWith('/work/here'))).toBe(true); + expect(probed.every(p => toPosix(p).startsWith('/work/here'))).toBe(true); }); }); diff --git a/src/version.ts b/src/version.ts index efa916d..f60fdfd 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.2.0'; +export const VERSION = '0.3.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index fcc3d85..62efe5d 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -133,20 +133,29 @@ exports[`--help snapshots > project 1`] = ` Manage TestSprite projects Options: - -h, --help display help for command + -h, --help display help for command Commands: - list [options] List projects visible to the API key + list [options] List projects visible to the API key Exit codes: 0 success 3 auth error 5 validation error (e.g., bad --page-size) 10 transport/network failure (UNAVAILABLE) — retry the command - get Get a project by id - create [options] Create a new project - update [options] Update project metadata - help [command] display help for command + get Get a project by id + create [options] Create a new project + update [options] Update project metadata + credential [options] Set the static backend credential injected + into every backend test + (Bearer token / API key / Basic token / + public). Free tier. + auto-auth [options] Configure the recurring-token + (auto-refresh login) for backend tests + (Pro). + A fresh token is fetched on each run and + injected into every backend test. + help [command] display help for command " `; @@ -242,7 +251,7 @@ Commands: Note: a 404 "not found" response is counted as skipped in the summary, not an error. run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. - Use --all --project for a wave-ordered batch run of all BE tests (M4). + Use --all --project for a wave-ordered batch run of all tests in a project (M4). Exit codes: 0 passed (or queued without --wait) @@ -362,7 +371,9 @@ Write a self-contained failure-context bundle for a test's latest failing run Options: --out Directory to write the §7 disk layout into (default: print wire envelope to stdout) - --failed-only Keep only the failed step plus its immediate neighbors (±1) + --failed-only Trim to the failed step ±1. The bundle is already + failure-focused server-side, so this is usually a no-op; use + \`test steps \` for the full run trail. -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -548,7 +559,7 @@ exports[`--help snapshots > test run 1`] = ` "Usage: testsprite test run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. -Use --all --project for a wave-ordered batch run of all BE tests (M4). +Use --all --project for a wave-ordered batch run of all tests in a project (M4). Exit codes: 0 passed (or queued without --wait) @@ -572,9 +583,9 @@ Options: 600) --idempotency-key opaque key for safe retries (1–256 chars). Printed to stderr at --debug if auto-generated. - --all run all BE tests in the project (wave-ordered - fresh run; requires --project). Mutually - exclusive with . (default: false) + --all run all tests in the project (wave-ordered fresh + run; requires --project). Mutually exclusive with + . (default: false) --project project id (required with --all; returned by \`testsprite project list\`) --filter with --all: only run tests whose name contains @@ -589,13 +600,17 @@ Options: -h, --help display help for command Dependency-aware fresh run (M4): - testsprite test run --all --project run all BE tests in wave order + testsprite test run --all --project run all project tests in wave order testsprite test run --all --project --filter name-glob subset testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). +Frontend tests: the current unified engine runs FE tests too (they are billed +like any run). On the legacy backend-only engine FE tests cannot run — they are +reported under skippedFrontend with an advisory; run those with 'test run '. + Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " @@ -640,10 +655,13 @@ Options: without the full trace. --debug Print HTTP method/path, request id, latency, retry decisions to stderr - --dry-run Skip the network, credentials, and filesystem; - emit a canned sample matching the OpenAPI - contract. Useful for learning the CLI surface - without an API key. + --dry-run Skip the network and credentials; emit a canned + sample matching the OpenAPI contract. Useful for + learning the CLI surface without an API key. + Note: file inputs you pass + (--plan-from/--plans/--steps) are still read and + validated locally; only --code-file uses a + placeholder. --request-timeout Client-side per-request timeout in seconds (default: 120). Aborts any single fetch that does not complete within this deadline. Override diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index b0537b1..959f212 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -8,7 +8,7 @@ */ import { execFileSync, spawn } from 'node:child_process'; -import { existsSync, mkdtempSync, statSync } from 'node:fs'; +import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; @@ -364,7 +364,10 @@ function runCli(args: string[], envOverrides: Record = {}): Prom cwd: REPO_ROOT, env: { ...process.env, + // os.homedir() reads HOME on POSIX but USERPROFILE on Windows — + // set both so the child never sees the real ~/.testsprite. HOME: tmpHome, + USERPROFILE: tmpHome, TESTSPRITE_API_KEY: undefined, TESTSPRITE_API_URL: undefined, ...envOverrides, @@ -897,7 +900,10 @@ describe('setup --from-env subprocess', () => { expect(result.exitCode).toBe(0); const credentialsPath = join(tmpHome, '.testsprite', 'credentials'); expect(existsSync(credentialsPath)).toBe(true); - expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); + // POSIX file modes don't exist on Windows (stat reports 0666). + if (process.platform !== 'win32') { + expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); + } }, 30_000); it('exits 5 with VALIDATION_ERROR when --from-env is set without TESTSPRITE_API_KEY', async () => { @@ -1044,7 +1050,7 @@ describe('--dry-run subprocess smoke', () => { // skipped the prompt. const credPath = join(tmpHome, '.testsprite', 'credentials'); // Make sure any previous test didn't leave one behind. - if (existsSync(credPath)) execFileSync('rm', [credPath]); + rmSync(credPath, { force: true }); const result = await runCli(['setup', '--dry-run', '--no-agent', '--output', 'json']); expect(result.exitCode).toBe(0); expect(existsSync(credPath)).toBe(false); diff --git a/test/helpers/hermetic-env.ts b/test/helpers/hermetic-env.ts new file mode 100644 index 0000000..c798bb3 --- /dev/null +++ b/test/helpers/hermetic-env.ts @@ -0,0 +1,41 @@ +/** + * Unit-test env hermeticity (vitest `setupFiles`, runs before each test file). + * + * Two leaks this closes, both of which made results depend on the + * developer's machine: + * + * 1. Real `TESTSPRITE_*` env vars. `loadConfig` gives `TESTSPRITE_API_KEY` + * precedence over the credentials file, so a key exported in the + * developer's shell silently overrode test fixtures. + * 2. The real home directory. `os.homedir()` reads `HOME` on POSIX but + * `USERPROFILE` on Windows, so the documented `HOME=$(mktemp -d)` + * recipe never isolated Windows runs. Both vars are redirected to a + * throwaway dir so no test can read or write `~/.testsprite`. + * + * Tests that need these vars set them explicitly (on `process.env` or via + * injected `env` deps) after this runs. + */ +import { existsSync, mkdirSync, mkdtempSync } from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const realHome = homedir(); +const hermeticHome = mkdtempSync(join(tmpdir(), 'testsprite-unit-home-')); +if (process.platform === 'win32') { + // Node version shims (Volta) resolve LocalAppData under USERPROFILE and + // abort if it's missing, which would break the `npm run build` beforeAll + // in the subprocess/snapshot suites. + mkdirSync(join(hermeticHome, 'AppData', 'Local'), { recursive: true }); +} +// Same shim concern on macOS/Linux: Volta derives ~/.volta from HOME unless +// VOLTA_HOME is set. Pin it to the real install before redirecting HOME. +const realVoltaHome = join(realHome, '.volta'); +if (!process.env.VOLTA_HOME && existsSync(realVoltaHome)) { + process.env.VOLTA_HOME = realVoltaHome; +} +process.env.HOME = hermeticHome; +process.env.USERPROFILE = hermeticHome; + +for (const key of Object.keys(process.env)) { + if (key.startsWith('TESTSPRITE_')) delete process.env[key]; +} diff --git a/vitest.config.ts b/vitest.config.ts index add8f0e..bd9b2ad 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,9 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], + // Strip real TESTSPRITE_* env vars and redirect the home dir so results + // never depend on the developer's shell or ~/.testsprite (see the file). + setupFiles: ['./test/helpers/hermetic-env.ts'], // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). fileParallelism: false, From e91c5a107867f16ca183b11684c84d953d972cbf Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Thu, 16 Jul 2026 14:56:49 -0700 Subject: [PATCH 063/117] release: v0.4.0 Private-Snapshot-RevId: afa3bf501545df536cfa638a86ae096fb7c34880 --- .github/workflows/backport-dispatch.yml | 83 ++ .github/workflows/ci-nudge.yml | 30 +- .github/workflows/ci.yml | 64 ++ .github/workflows/issue-triage.yml | 12 +- .github/workflows/pr-triage.yml | 24 +- .github/workflows/release.yaml | 28 +- .github/workflows/test-coverage.yml | 2 + .gitleaks.toml | 19 + CHANGELOG.md | 21 + CONTRIBUTING.md | 27 +- DOCUMENTATION.md | 74 +- README.md | 59 +- package-lock.json | 4 +- package.json | 2 +- scripts/README.md | 29 + skills/testsprite-verify.codex.md | 6 + skills/testsprite-verify.skill.md | 19 +- src/commands/agent.test.ts | 156 ++-- src/commands/auth.test.ts | 57 ++ src/commands/auth.ts | 14 + src/commands/doctor.test.ts | 45 + src/commands/doctor.ts | 60 +- src/commands/project.test.ts | 168 +++- src/commands/project.ts | 130 ++- src/commands/test.cancel.spec.ts | 309 +++++++ src/commands/test.quickwins.spec.ts | 110 +++ src/commands/test.rerun.spec.ts | 98 ++- src/commands/test.run.spec.ts | 162 +++- src/commands/test.test.ts | 103 +++ src/commands/test.ts | 815 ++++++++++++++++-- src/commands/test.wait.spec.ts | 122 ++- src/index.ts | 45 +- src/lib/bundle.test.ts | 19 + src/lib/bundle.ts | 19 +- src/lib/client-factory.ts | 23 + src/lib/dry-run/samples.test.ts | 33 + src/lib/dry-run/samples.ts | 43 +- src/lib/errors.test.ts | 22 + src/lib/errors.ts | 45 + src/lib/failing-fe-resolver.spec.ts | 193 +++++ src/lib/failing-fe-resolver.ts | 67 +- src/lib/http.test.ts | 246 +++++- src/lib/http.ts | 133 ++- src/lib/interrupt.test.ts | 101 ++- src/lib/interrupt.ts | 124 ++- src/lib/poll.spec.ts | 104 ++- src/lib/poll.ts | 95 +- src/lib/runs.types.ts | 14 + src/lib/v3-advisory.test.ts | 24 + src/lib/v3-advisory.ts | 25 + src/lib/version-notice.test.ts | 110 +++ src/lib/version-notice.ts | 111 +++ src/version.ts | 2 +- test/__snapshots__/help.snapshot.test.ts.snap | 72 +- test/cli.subprocess.test.ts | 5 +- test/e2e/signal.e2e.test.ts | 188 ++++ test/help.snapshot.test.ts | 5 +- test/helpers/execNpm.ts | 17 + 58 files changed, 4410 insertions(+), 327 deletions(-) create mode 100644 .github/workflows/backport-dispatch.yml create mode 100644 scripts/README.md create mode 100644 src/commands/test.cancel.spec.ts create mode 100644 src/lib/v3-advisory.test.ts create mode 100644 src/lib/v3-advisory.ts create mode 100644 src/lib/version-notice.test.ts create mode 100644 src/lib/version-notice.ts create mode 100644 test/e2e/signal.e2e.test.ts create mode 100644 test/helpers/execNpm.ts diff --git a/.github/workflows/backport-dispatch.yml b/.github/workflows/backport-dispatch.yml new file mode 100644 index 0000000..8b6a99a --- /dev/null +++ b/.github/workflows/backport-dispatch.yml @@ -0,0 +1,83 @@ +# backport-dispatch.yml — OPTIONAL latency optimization for DEV-352's +# auto-backport bot. Authored in atlas (release/public path), ships to the +# PUBLIC repo via the snapshot per invariant I0 — this workflow only ever +# runs on the public repo, never on atlas (see the repo guard below). +# +# NOT YET ARMED (2026-07-15): the atlas-side auto-backport.yml has a +# scheduled SWEEP (every 30 min) that scans merged public PRs anonymously +# and backports anything unabsorbed — the whole pipeline is fully +# functional with ZERO public-repo credentials via that path alone. This +# workflow instantly notifies atlas the moment a PR merges instead of +# waiting for the next sweep (seconds vs. up to 30 minutes) — a nice-to-have, +# not a requirement. +# +# Arming it means placing a GitHub App credential with contents:write on +# the PRIVATE atlas repo into THIS PUBLIC repo's secrets — a real trust +# decision (a compromised public-repo secret store would let an attacker +# push to atlas) that is deliberately left to the operator, not decided by +# this file. Until TESTSPRITE_HOB_APP_ID/TESTSPRITE_HOB_PRIVATE_KEY (or the +# ALFHEIM_AGENT_* fallback) exist on the PUBLIC repo with contents:write on +# atlas, this workflow no-ops cleanly (see the "not armed" step) — it does +# NOT fail loud, so it stays quiet for every contributor watching the +# Actions tab until the operator decides to enable it. +# +# SECURITY: this workflow reads ONLY event metadata (PR number, merge SHA, +# base ref) — it never checks out the merged PR's code. That is what makes +# `pull_request_target` safe here: the base-repo context (secrets, a +# write-capable token) is required because a fork-originated merged PR gets +# ZERO secrets under a plain `pull_request` trigger, even for the `closed` +# activity type — but nothing in this job ever runs untrusted fork code, so +# the classic pull_request_target RCE/secret-exfiltration risk (checking +# out and executing the fork's own head ref under a privileged context) +# does not apply. Do not add actions/checkout to this workflow. +name: Notify atlas of a merged public PR + +on: + pull_request_target: + types: [closed] + +permissions: + contents: read + +jobs: + notify: + if: github.repository == 'TestSprite/testsprite-cli' && github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + owner: TestSprite + repositories: testsprite-cli-atlas + # Minimum required for POST /repos/{owner}/{repo}/dispatches (verified + # against docs.github.com/en/rest/using-the-rest-api/permissions-required-for-github-apps — + # the endpoint is gated on Contents:write, not Contents:read). + permission-contents: write + + - name: Dispatch to atlas + if: steps.app-token.outputs.token != '' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + # These two fields are fork-influenced (merge_commit_sha/base.ref come + # from a merged PR's event payload) — routed through env instead of + # spliced into the run: string to avoid the GHA script-injection class + # (a maliciously-crafted value could otherwise break out of the -F + # argument and inject arbitrary shell). PR_NUMBER is left inline: it is + # a GitHub-assigned integer, not attacker-authorable content. + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + gh api repos/TestSprite/testsprite-cli-atlas/dispatches \ + -f event_type=public-pr-merged \ + -F 'client_payload[pr_number]=${{ github.event.pull_request.number }}' \ + -F "client_payload[merge_commit_sha]=$MERGE_SHA" \ + -F "client_payload[base_ref]=$BASE_REF" + + - name: Not armed yet — the atlas sweep will pick this up + if: steps.app-token.outputs.token == '' + run: | + echo "::notice::No cross-repo credential configured on this repo yet — atlas's scheduled sweep (auto-backport.yml, every 30 min) will pick up this merge instead of an instant dispatch. See DEV-352 / DEV-348 / DEV-350 for the operator decision to arm this." diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml index d01ede3..f14461f 100644 --- a/.github/workflows/ci-nudge.yml +++ b/.github/workflows/ci-nudge.yml @@ -9,8 +9,10 @@ # CI workflows (CI + Test Coverage) can complete in any order. # # Bot identity: same App-token-first / GITHUB_TOKEN-fallback pattern as -# pr-triage.yml (the App needs the "Pull requests: Read & write" permission to -# post as testsprite-hob[bot]; until then comments come from github-actions[bot]). +# pr-triage.yml. The App has had the "Pull requests: Read & write" permission +# since 2026-07-02 (corrected 2026-07-15; this comment previously said the +# permission was still pending) — the fallback path stays as defense-in-depth +# for whenever the App/secrets are absent. name: CI failure nudge on: @@ -40,8 +42,20 @@ jobs: with: app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # The script below only ever calls the App client (`appClient`) for + # rest.issues.updateComment/createComment — PR comments ride the + # issues API (see the header comment above). All reads (checks, + # pulls, commit->PR lookup) go through the ambient `github` client, + # governed by this workflow's top-level `permissions:` block, not + # this minted token. Scoping to checks:read/pull-requests:write here + # (matching the job's top-level block literally) would risk the mint + # itself failing if the App's installation doesn't hold Checks + # permission — which would silently drop the App-token path + # entirely (continue-on-error swallows the failure) and always fall + # back to github-actions[bot], defeating this bot's own purpose. + permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: APP_TOKEN: ${{ steps.app-token.outputs.token }} with: @@ -82,14 +96,20 @@ jobs: 'Build': 'run `npm run build` and fix the compile errors', 'Local E2E Tests': 'run `npm run test:e2e` (it builds first)', 'Coverage (>= 80%)': 'run `npm run test:coverage` — new code needs tests until every metric is back at 80%', + 'Secret scan (gitleaks)': 'run `gitleaks detect --no-git --redact --source .` locally (config: .gitleaks.toml) and remove the secret — or, for a provable false positive, add a narrowly-anchored allowlist regex', }; + // Matrix jobs publish check runs as "Unit Tests (Node 20)" etc. — + // exact lookup alone would silently skip them (and the nudge would + // say "all green" while they are red). Try exact first, then the + // name with a trailing parenthetical stripped (review round 2). + const fixFor = (name) => FIX[name] ?? FIX[name.replace(/\s*\([^)]*\)$/, '')]; // Recompute full CI state from this SHA's check runs, narrowed to the // CI jobs above — other workflows (e.g. pr-triage) also attach // github-actions check runs to the PR head and must not count here. const checks = await github.paginate(github.rest.checks.listForRef, { owner, repo, ref: run.head_sha, per_page: 100 }); - const ours = checks.filter(c => c.app && c.app.slug === 'github-actions' && FIX[c.name]); + const ours = checks.filter(c => c.app && c.app.slug === 'github-actions' && fixFor(c.name)); const failing = ours.filter(c => ['failure', 'timed_out'].includes(c.conclusion)); const pending = ours.filter(c => c.status !== 'completed'); @@ -110,7 +130,7 @@ jobs: const lines = failing .sort((a, b) => a.name.localeCompare(b.name)) .map(c => { - const fix = FIX[c.name] || 'see the logs for details'; + const fix = fixFor(c.name) || 'see the logs for details'; return `- **${c.name}** — ${fix} ([logs](${c.html_url}))`; }); const body = `${marker}\nThanks, @${pr.user.login}! CI is red on this PR — ` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edde076..3766b0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -37,6 +39,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -55,6 +59,8 @@ jobs: node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -68,6 +74,27 @@ jobs: env: CI: true + test-windows: + name: Unit Tests (Windows) + runs-on: windows-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + - run: npm run build + - run: npm run typecheck + - run: npm test + env: + CI: true + build: name: Build (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest @@ -76,6 +103,8 @@ jobs: node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -96,6 +125,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -107,3 +138,36 @@ jobs: - run: npm run test:e2e env: CI: true + + gitleaks: + name: Secret scan (gitleaks) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + # Same pinned version + checksum-verified install as + # divergence-sentinel.yml (the more recent/secure of the two gitleaks + # invocations already in this repo — release-build.yml pins an older + # 8.21.2 with no checksum check). Continuous, per-PR/per-push gate: a + # working-tree scan (--no-git), not a full-history scan — history + # scanning is too slow to run on every PR, and this mirrors the mode + # scripts/make-public-snapshot.sh already uses for the release-time scan + # (`gitleaks detect --no-git --no-banner --redact --source `). + - name: Install gitleaks + env: + GITLEAKS_VERSION: '8.28.0' + run: | + set -euo pipefail + BASE="https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}" + TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -sSL -o "$TARBALL" "${BASE}/${TARBALL}" + curl -sSL -o checksums.txt "${BASE}/gitleaks_${GITLEAKS_VERSION}_checksums.txt" + grep " ${TARBALL}\$" checksums.txt | sha256sum -c - + tar -xzf "$TARBALL" gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Scan working tree for secrets + run: gitleaks detect --no-git --no-banner --redact --source . diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 6b6ce62..0da78c8 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -31,10 +31,18 @@ env: jobs: triage: + # Public repo only — this file also lives in the private mirror (atlas), + # where issue-first claiming doesn't apply; the three sibling bots + # (pr-triage.yml, ci-nudge.yml, stale.yml) all carry this same fence and + # this one was missing it (found in the 2026-07-15 OSS-P2 audit — the + # /assign bot was live on atlas too until this fix). # issues only (issue_comment also fires on PRs), and never react to a bot's own # comment — incl. our own testsprite-hob[bot], which (unlike GITHUB_TOKEN) would # otherwise re-trigger this workflow. `type == 'Bot'` covers both bot identities. - if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' }} + if: >- + github.repository == 'TestSprite/testsprite-cli' && + !github.event.issue.pull_request && + github.event.comment.user.type != 'Bot' runs-on: ubuntu-latest steps: # Mint a token for the testsprite-hob App so comments post as testsprite-hob[bot]. @@ -49,7 +57,7 @@ jobs: # of inheriting the App's full installation permissions. permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: # App token when available (→ testsprite-hob[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml index a9cd23d..cfd114d 100644 --- a/.github/workflows/pr-triage.yml +++ b/.github/workflows/pr-triage.yml @@ -13,12 +13,12 @@ # Runs with NO checkout — metadata-only, so it is safe under pull_request_target # (which is required for fork PRs to get a write-capable token). # -# Bot identity: posts as `testsprite-hob[bot]` when the App token works. The -# App currently has Issues R/W + Metadata only — commenting/labeling a PULL -# REQUEST needs the "Pull requests: Read & write" App permission (issues-API -# endpoints are permission-checked by target type). Until an org admin adds -# that permission and re-approves the installation, every call gracefully falls -# back to the default GITHUB_TOKEN and posts as github-actions[bot]. +# Bot identity: posts as `testsprite-hob[bot]` when the App token works (the +# App has had the "Pull requests: Read & write" permission since 2026-07-02 — +# corrected 2026-07-15; this comment previously said the permission was still +# pending). The App-token-first / GITHUB_TOKEN-fallback code path below is +# kept regardless, as defense-in-depth for whenever the App/secrets are +# absent or a future installation loses the permission. # Secrets: TESTSPRITE_HOB_* preferred; the ALFHEIM_AGENT_* fallbacks are the # original names from before the App was renamed (same App ID + key). name: PR triage (issue-link gate) @@ -60,8 +60,18 @@ jobs: with: app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # Every write() call below (createComment/updateComment/addLabels/ + # removeLabel) is an `issues.*` Octokit method — GitHub gates all + # four (comments AND labels, even on a PR number) on the "Issues" + # repository permission, not "Pull requests" (verified against + # docs.github.com/en/rest/using-the-rest-api/permissions-required-for-github-apps). + # Scoping to issues:write alone is therefore both sufficient and + # minimal — narrower than this job's top-level `permissions:` block, + # which also carries pull-requests:write for the ambient-token + # fallback path (unused by this specific App-token mint). + permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: APP_TOKEN: ${{ steps.app-token.outputs.token }} with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a0f1346..8ef6dc9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,13 +14,33 @@ jobs: with: node-version: 22 registry-url: 'https://registry.npmjs.org' - # npm trusted publishing (OIDC) requires npm >= 11.5.1; Node 22 bundles npm 10.x - - run: npm install -g npm@latest + # npm trusted publishing (OIDC) requires npm >= 11.5.1; Node 22 never bundles + # that (it ships 10.9.x). Pin an EXACT version rather than floating on + # @latest: npm 12.0.0 shipped with a broken workspace symlink that pulled the + # wrong `sigstore` dependency into its own release tarball, so every + # `npm publish --provenance` failed with `Cannot find module 'sigstore'` + # (npm/cli#9722) — this is exactly what our v0.2.0 and v0.3.0 release.yaml + # runs hit, both AFTER `npm install -g npm@latest` reported success. Fixed + # in 12.0.1 (2026-07-10); bump this pin deliberately, never float it again. + - run: npm install -g npm@11.6.2 - run: npm ci - run: npm run lint - run: npm run typecheck - run: npm run format:check - run: npm run test:coverage - run: npm run build - # Auth via npm trusted publishing (OIDC) — no token needed; provenance is implied - - run: npm publish --provenance --access public + # Auth via npm trusted publishing (OIDC) — no token needed; provenance is + # implied. A prerelease tag (e.g. v0.3.0-rc.1, contains "-") must publish + # under an explicit dist-tag: current npm already refuses an implicit + # `latest` for a prerelease version, but failing mid-release is worse than + # not needing the guard, so this still passes --tag itself rather than + # relying on that enforcement alone. + - name: Publish + env: + REF_NAME: ${{ github.ref_name }} + run: | + if [[ "$REF_NAME" == *-* ]]; then + npm publish --provenance --access public --tag next + else + npm publish --provenance --access public + fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index ebb3331..0e20d9e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -16,6 +16,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 diff --git a/.gitleaks.toml b/.gitleaks.toml index b3f68df..b31d49c 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -19,4 +19,23 @@ regexes = [ '''sk-secret-[0-9]+''', # Documented placeholder keys used by dry-run/sample responses '''sk-user-(test|DRY-RUN)''', + # Example values inside internal (never-shipped, dropped-before-publish) + # design docs. These are only ever scanned by ci.yml's continuous gitleaks + # job (new, 2026-07) — the release-time scan + # (scripts/make-public-snapshot.sh) already drops those internal docs + # wholesale before it ever runs gitleaks, so these never previously + # tripped a scan. Deliberately described here without naming the internal + # doc paths themselves (this file ships to the public repo, and the + # snapshot's own internal-doc-reference scan would flag such a path). + # A "Bearer " curl example whose placeholder token literal is the + # word "dev-token" — not a real key. (Allowlist regexes match against the + # extracted secret value, not the surrounding line — verified empirically + # with `gitleaks detect`.) + '''^dev-token$''', + # A literal base64 pagination-cursor EXAMPLE value in an internal OpenAPI + # spec (decodes to {"ek":"project_a47b2c11"} — spec-example data, not a + # credential). Anchored ^…$ so it only ever matches this exact literal — + # an unanchored version would also suppress any REAL secret that happens + # to contain this substring (review round 2, 2026-07-15; reproduced). + '''^eyJlayI6InByb2plY3RfYTQ3YjJjMTEifQ==$''', ] diff --git a/CHANGELOG.md b/CHANGELOG.md index e78347e..45803e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +## [0.4.0] - 2026-07-16 + +### Added + +- **`test cancel `** — user-initiated cancel of in-flight runs (the real stop button; Ctrl-C only detaches). A single id renders the run card with status `cancelled` (plus an advisory when it was already cancelled); multiple ids print a `{cancelled, alreadyCancelled, conflicts, notFound}` summary. Exit codes: 4 when any id is not found, else 6 on conflicts, else 0. `--dry-run` supported. +- **Graceful Ctrl-C during `--wait`** — SIGINT/SIGTERM now detaches cleanly instead of killing the process mid-poll: the in-flight request aborts immediately, stdout gets the same partial `{runId, status: "running"}` envelope as the request-timeout path, and stderr states the truth — the server-side run keeps executing (and billing) — with a re-attach hint and a `test cancel` pointer. Exit 130/143/129 per the documented signal contract; a second signal forces a hard exit. Interrupting never cancels the server-side run — that's what `test cancel` is for. +- **`project delete --confirm`** — permanently delete a project and everything under it (its frontend/backend sub-projects, all their tests, and backend fixtures), mirroring the Portal's cascade delete. Requires `--confirm` (the CLI never prompts); `--dry-run` previews the response shape without a network call. Standard exit codes: 0 success, 3 auth, 4 not-found (or already-deleted), 5 validation (e.g. missing `--confirm`). +- **Backend stdout and traceback in results** — `test result` and the failure bundle now surface the backend test's captured stdout and Python traceback: full content in `--output json` (and in `result.json` / `failure.json` bundle files), and a bounded 20-line tail with a byte count in text mode. No change for frontend tests, passing runs, or older backends. +- **Backend dependency declarations are now readable and editable** — `test get` surfaces `produces` / `consumes` / `category`, and `test update` accepts `--produces` / `--needs` / `--category` (previously create-only). +- **Version-compatibility handshake** — the CLI reads the backend's advertised minimum-supported-version on every response and prints a one-line upgrade advisory on stderr when the running version is below the floor (honors the same opt-outs as the update notice; never alters exit status). A `CLIENT_TOO_OLD` rejection (HTTP 426) is now a first-class error: exit 14, non-retriable, rendered with upgrade guidance and the version gap. +- **V3 routing visibility** — `auth status` and `doctor` render a `routing: v2|v3` line when the backend reports the account's routing, and V3-routed accounts get one consolidated advisory listing the known V3-path behavior gaps. Text mode only — JSON consumers read `v3Enabled` from the `/me` payload; absent-safe against older backends. + +### Changed + +- **The `testsprite-verify` agent skill routes local-only changes to the TestSprite MCP** — the skill now states the reachability gate explicitly: the CLI verifies reachable deployed URLs only; when the change is only running locally, the skill hands off to the TestSprite MCP when available (an explicitly named tool always wins), instead of failing against localhost. + +### Fixed + +- **`project create --description` now fails fast with a clear validation error** — projects have no description field, so the flag's value was previously dropped silently; the error points at test-level descriptions (`test create --description`) instead. +- **Standalone backend run cards no longer show a misleading step summary** — `test run` / `test wait` / `test rerun` cards for backend tests render `steps: n/a (backend)` instead of `0/0 (passed=0, failed=0)` (backend tests have no per-step storage). + ## [0.3.0] - 2026-07-08 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bed91fc..1479099 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,11 +109,36 @@ npm run format:check # Prettier (check only) npm run typecheck # tsc --noEmit ``` +## Developing on Windows + +Native Windows (no WSL, no Git Bash required) is a fully supported dev +environment — you only need **git** and **Node ≥ 20**. `npm ci`, `npm run +build`, `npm test`, `npm run lint`, and `npm run typecheck` all run the same +way as on macOS/Linux, and CI runs the unit suite on `windows-latest` as the +reference environment (see [CI gates](#ci-gates-required-for-merge) below) — +if it's green there, it's green on your machine. + +A couple of things worth knowing: + +- Line endings are normalized to LF on checkout via `.gitattributes` + (`core.autocrlf` doesn't need any special local configuration). +- `--out`/bundle-path flags accept native Windows paths (`C:\Users\...`) + including a trailing backslash and a bare drive root (`C:\`). +- A small number of tests that create real filesystem symlinks are skipped on + Windows (creating a symlink there needs Administrator rights or Developer + Mode, which isn't guaranteed on hosted CI) — the safety behavior they cover + is still exercised on the Linux/macOS CI jobs. + +Hit something that doesn't work on Windows? Please file it — see +[Questions & support](#questions--support) above, and feel free to tag it +`good first issue` if it looks like an isolated fix; we'd rather know than +have you route around it silently. + ## CI gates (required for merge) - ESLint + Prettier clean - TypeScript type-check clean -- All unit tests passing +- All unit tests passing on Linux (Node 20 + 22) **and** on Windows (`windows-latest`, Node 22) - Coverage ≥ 80% on lines / statements / functions / branches - Build + smoke test of the CLI binary diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 2f6cd6c..bf6b1b5 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -180,7 +180,7 @@ Common flags: #### `testsprite test get ` -Get a single test by id. Test ids look like `test_xxxxxxxx` and come from `test list`. +Get a single test by id. Test ids look like `test_xxxxxxxx` and come from `test list`. Backend tests echo their dependency declarations — `produces` / `consumes` / `category` — when present. ```bash testsprite test get test_xxxxxxxx --output json @@ -210,7 +210,7 @@ Common flags: `--page-size`, `--starting-token`, `--max-items` — same shape as #### `testsprite test result ` -Get the latest result for a test — status, started / finished timestamps, video and failure-analysis URLs, summary counts (`passed / failed / skipped`), and correlation fields (`snapshotId`, `runId`, `codeVersion`). With `--include-analysis`, the response also carries an inline `analysis` block (root-cause hypothesis, recommended fix target, failure kind). +Get the latest result for a test — status, started / finished timestamps, video and failure-analysis URLs, summary counts (`passed / failed / skipped`), and correlation fields (`snapshotId`, `runId`, `codeVersion`). With `--include-analysis`, the response also carries an inline `analysis` block (root-cause hypothesis, recommended fix target, failure kind). Backend tests additionally surface the run's captured stdout (`apiOutput`) and Python traceback (`trace`): full content under `--output json` (and in `result.json` / `failure.json` inside failure bundles); text mode prints a bounded 20-line tail of each with a byte count. ```bash testsprite test result test_xxxxxxxx --output json @@ -293,7 +293,7 @@ testsprite test lint --steps ./refined.plan.json # the shape `test plan pu #### `testsprite test create` -Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. +Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. Backend tests can declare wave-ordering dependencies at create time — `--produces ` / `--needs ` (repeatable) and `--category ` — and amend them later via `test update`. ```bash # Backend test from a code file @@ -319,10 +319,11 @@ testsprite test create-batch --plan-from-dir ./plans/ --dry-run --output json #### `testsprite test update ` -Update test metadata (name, description). +Update test metadata (name, description, priority) — and, for **backend tests**, the dependency declarations: `--produces ` / `--needs ` (repeatable) and `--category `. Updated declarations are echoed back by `test get`. ```bash testsprite test update test_xxxxxxxx --name "Renamed test" --description "Updated" +testsprite test update test_be_xxxx --produces session_token --category setup testsprite test update test_xxxxxxxx --dry-run --output json ``` @@ -358,13 +359,22 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Note the asymmetry: `--description` is **create-only** — `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`, but not `--description`. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Projects have **no description field** — `--description` is rejected client-side with a validation error (descriptions live on tests: `test create --description`). `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`. ```bash testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` +#### `testsprite project delete ` + +Permanently delete a project and **everything under it** — its frontend/backend sub-projects, all their tests, and backend fixtures (mirrors the Portal's cascade delete). There is **no restore window**. `--confirm` is required (the CLI never prompts); absent it, the CLI exits 5 with a local validation error. `--dry-run` previews the response shape without a network call. Exit codes: 0 success, 3 auth, 4 not found (or already deleted), 5 validation. + +```bash +testsprite project delete proj_xxxxxxxx --confirm +testsprite project delete proj_xxxxxxxx --dry-run --output json +``` + #### `testsprite project credential ` Set the **static backend credential** injected into every backend test in the project (free tier). Supported types: `public` (no credential), `"Bearer token"`, `"API key"`, `"basic token"`. @@ -521,6 +531,16 @@ testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +#### `testsprite test cancel ` + +Cancel one or more in-flight runs — the counterpart to Ctrl-C, which only **detaches** (the server-side run keeps executing and billing). Cancelling is idempotent: an already-cancelled run reports `alreadyCancelled` as an advisory, not an error; a run that already reached a terminal verdict is a conflict — the verdict is never overwritten, and no credits are refunded. With one id, prints the run card; with several, prints a `{ cancelled, alreadyCancelled, conflicts, notFound }` summary. Exit codes: any unknown id → 4; else any conflict → 6; else 0. + +```bash +testsprite test cancel run_01hx3z9p8q4k2y7a +testsprite test cancel run_aaaa run_bbbb --output json +testsprite test cancel run_01hx3z9p8q4k2y7a --dry-run --output json +``` + #### `testsprite test artifact get ` Download the failure bundle for a specific `runId`. Same on-disk layout as `test failure get`, but addressed by `runId` instead of `testId`, so an agent can fetch the bundle for the exact run it just triggered — never a newer failure on the same test. Default `` is `./.testsprite/runs//`. The CLI enforces `meta.runId === ` as an integrity check; a mismatch exits 5 rather than silently writing the wrong bundle. @@ -603,6 +623,13 @@ check is skipped in CI, when stderr is not a TTY, under `--output json` / failure is silent: the notice can never break or delay a command. This is the only outbound call the CLI makes besides your configured API endpoint. +Separately, the backend advertises its **minimum supported CLI version** on +every `/api/cli/v1` response. When the running CLI is below that floor, a +one-line upgrade advisory is printed to stderr (same opt-outs as the update +notice; it never changes the exit status). A backend may also reject a +too-old client outright with HTTP 426 — surfaced as `CLIENT_TOO_OLD`, +exit `14`, non-retriable, with upgrade guidance. + ### Scopes API-key scopes gate the write and run surfaces: @@ -613,8 +640,8 @@ API-key scopes gate the write and run surfaces: | `read:projects` | `project list / get` | | `read:tests` | every `test *` read command | | `write:tests` | `test create / create-batch / update / delete / code put / plan put` | -| `write:projects` | `project create / update / credential / auto-auth` | -| `run:tests` | `test run / rerun / flaky / wait / artifact get` | +| `write:projects` | `project create / update / delete / credential / auto-auth` | +| `run:tests` | `test run / rerun / flaky / wait / cancel / artifact get` | New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. @@ -632,25 +659,26 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not ## Exit codes -| Code | Meaning | -| --------------------- | --------------------------------------------------------------------------- | -| `0` | Success | -| `1` | Generic failure / non-passed run status | -| `2` | Not yet implemented | -| `3` | Auth error | -| `4` | Not found | -| `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | -| `7` | Timeout / unsupported | -| `10` | Service unavailable | -| `11` | Rate limited (retriable) | -| `12` | Insufficient credits (non-retriable) | -| `13` | Feature gated (paid plan required) | -| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | +| Code | Meaning | +| --------------------- | ------------------------------------------------------------------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | +| `14` | Client too old — the backend requires a newer CLI (HTTP 426 `CLIENT_TOO_OLD`); upgrade to proceed | +| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | ### Signals & pipes -On SIGINT (Ctrl-C), SIGTERM, or SIGHUP the CLI prints `Interrupted (). Any run already started keeps executing on the server; check it with 'testsprite test list' or 'testsprite test wait '.` and exits `128 + signal`. **Ctrl-C does not cancel the server-side run** — execution (and any credit spend) continues; there is no cancel command today, so re-attach with `test wait ` instead of re-triggering. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. +During any `--wait`, SIGINT (Ctrl-C), SIGTERM, or SIGHUP triggers a **graceful detach**: the in-flight request aborts immediately, stdout gets the same partial `{ runId, status: "running" }` envelope as the request-timeout path (under `--output json`, stderr carries an `INTERRUPTED` envelope naming the signal), and stderr states the truth — the server-side run keeps executing, and any credit spend continues — with a re-attach hint (`test wait `) and a `test cancel ` pointer. The exit code is `128 + signal` (130 / 143 / 129). A second signal forces an immediate hard exit. Outside a `--wait` (prompts, one-shot commands), signals keep the pre-existing immediate-exit behavior. **Ctrl-C never cancels the server-side run** — `test cancel ` is the explicit stop. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. ## Design principles diff --git a/README.md b/README.md index 78d293a..3bb465e 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude > **Pointing a coding agent (Claude Code, Cursor, Codex, Cline, …) at TestSprite?** Have it run `testsprite setup` first — that installs the verification skill, so the agent knows how to create, run, and triage tests on its own (instead of guessing from this README). New here? Start with the **[getting-started overview](https://docs.testsprite.com/cli/getting-started/overview)**. -> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). +> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. The backend also advertises its minimum supported CLI version — a below-floor CLI prints a one-line upgrade advisory on stderr, and a too-old client may be rejected with exit 14 (`CLIENT_TOO_OLD`). Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). From there, the loop runs on its own — an example session, typed by the coding agent: @@ -91,34 +91,35 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | -| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | -| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / permanently delete (no restore window; `--confirm` required) | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | -| | `test wait` | Block on one or more `runId`s until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | +| Group | Command | What it does | +| --------- | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | +| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | +| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata and BE dependency declarations (`--produces` / `--needs` / `--category`) / permanently delete (no restore window; `--confirm` required) | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` / `project delete` | Manage projects; `delete` removes a project and everything under it (`--confirm` required, no restore window) | +| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | +| | `test wait` | Block on one or more `runId`s until terminal | +| | `test cancel` | Cancel one or more in-flight runs (Ctrl-C during `--wait` only detaches — `cancel` is the real stop) | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/package-lock.json b/package-lock.json index b254e0d..4461016 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.2.0", + "version": "0.3.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", diff --git a/package.json b/package.json index e78ce17..18336cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.4.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..7e17420 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,29 @@ +# `scripts/` — what runs where + +Every file here is classified below so nobody has to guess whether it's +safe/expected to run on a laptop, and so a Windows contributor knows exactly +what (if anything) they're missing. Per DEV-356 ("Windows-proof the +toolchain"): **the release/backport shell scripts are CI-only** — nothing in +this directory requires a human to run bash or perl locally. + +| File | Classification | Runs on | Notes | +| ------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `make-public-snapshot.sh` | **CI-ONLY** | the release pipeline's build job (`release-build.yml`) and the nightly divergence sentinel (dry-run mode) | Builds the scrubbed public snapshot. Never ships to the public repo (it's in its own DROP list). Not meant to be run by hand — see [`docs/internal/cli-oss/release-pipeline-ops.md`](../docs/internal/cli-oss/release-pipeline-ops.md) for the operator-facing flow. | +| `backport-public-pr.sh` | **CI-ONLY** (human-runnable for conflict recovery) | the auto-backport workflow, invoked per merged community PR | Also directly runnable by an operator resolving a cherry-pick conflict (`--record-only` after a manual fix) — that's an escape hatch, not the steady-state path. Requires `bash` + `gh` + `jq`; a Windows operator resolving a conflict does it via Git Bash/WSL, or asks another maintainer — this one script is the sole remaining bash dependency in the whole release flow. | +| `generate-version.mjs` | **HUMAN-RUN** (Node) | `npm run prebuild` / `npm run generate:version`, any OS | Pure Node — no shell-out, no bash/perl/BSD-vs-GNU assumptions. | +| `postbuild.mjs` | **HUMAN-RUN** (Node) | `npm run build`, any OS | Pure Node `fs` calls (sets the executable bit on `dist/index.js`; a no-op on Windows, which has no POSIX exec bit). | +| `p0-status-coverage.sql` | **DROPPED-INTERNAL** | ad hoc, by an operator, against Athena | Not part of any npm script or CI workflow. Never ships to the public repo (internal AWS resource references). | + +## The upshot for a Windows contributor / operator + +- **Local dev loop** (`npm ci`, `npm test`, `npm run build`, `npm run lint`, + `npm run typecheck`) needs only **git + Node ≥ 20** — nothing in this + directory runs as part of that loop. See CONTRIBUTING.md's "Developing on + Windows" section. +- **Releasing** does not require running `make-public-snapshot.sh` (or any + bash) on your own machine at all — it's a `workflow_dispatch` you trigger + and approve from a browser. See + [`docs/internal/cli-oss/release-pipeline-ops.md`](../docs/internal/cli-oss/release-pipeline-ops.md). +- The **only** scenario that still touches bash directly is resolving a rare + backport cherry-pick conflict by hand — everything else in the pipeline is + either Node or runs unattended in Actions. diff --git a/skills/testsprite-verify.codex.md b/skills/testsprite-verify.codex.md index f2c2456..5d28f16 100644 --- a/skills/testsprite-verify.codex.md +++ b/skills/testsprite-verify.codex.md @@ -10,6 +10,12 @@ docs/config and is about to be reported complete. Run after a feature or fix lands. Skip only for: docs-only edits, pure build/config changes, or when the repo has no TestSprite project linked. +The CLI only tests a reachable deployed URL (it rejects localhost). If the +change is only running locally, hand off to the TestSprite MCP when it's +available — it tunnels your local server; otherwise report the change as +unverified-because-undeployed and stop. If the user explicitly named a tool +(the CLI or the MCP), honor that over this reachability heuristic. + ## Core loop ### 1. Preflight diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index fe7c661..5020e42 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -21,6 +21,17 @@ Run the loop only once the change is live somewhere reachable (e.g. open the PR, let CI deploy the preview/staging environment) and pass that URL as `--target-url`. Running earlier verifies the previous build, not your change. +This CLI only tests a reachable deployed URL (it rejects localhost). If the +change is only running locally and isn't deployed anywhere reachable yet: + +- if the TestSprite MCP is available in this environment, hand off to it — it + tunnels your local server and tests the running app; +- otherwise report the change as unverified-because-undeployed and stop — don't + run against a stale deployment to manufacture a verdict. + +If the user explicitly named a tool (the CLI or the MCP), honor that over this +reachability heuristic. + ## When to skip The skip list is narrow: @@ -225,10 +236,10 @@ testsprite test create --type backend --project --name "fixture user ordering: trigger the set with `test run --all` (§4) and producers run before consumers, `teardown` last. Chaining `run A --wait && run B --wait` yourself loses the engine's variable passing and conflicts with concurrent runs. -- **Declarations are currently create-only.** `test update` cannot amend them and - `test get` / `test list` don't echo them back, so note what each test - produces/needs as you create it; changing the graph later means delete + - recreate. +- **Declarations are editable.** `test update` amends them (`--produces` / + `--needs` / `--category`, repeatable) and `test get` echoes them back — + still declare the graph at create time when you can, so wave ordering is + right on the first run. **Show the user the drafted plan / code before creating it** — creating writes to their project. One short confirmation; let them edit the tempfile first. diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 24d8ec6..7326aeb 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -1108,75 +1108,87 @@ describe('runInstall — default AgentFs (real disk)', () => { expect(readFileSync(abs, 'utf8')).toBe(content); }); - it('refuses to write through a symlinked parent dir (real disk) — exit 5', async () => { - const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-parent-')); - const outside = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-')); - // `.claude` is a real symlink to a directory outside the project root. - symlinkSync(outside, path.join(tmpRoot, '.claude'), 'dir'); - const { deps } = makeCapture(); - - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: ['claude'], - skills: ['testsprite-verify'], - force: false, - dir: tmpRoot, - }, - { ...deps }, - ); - } catch (err) { - thrown = err; - } - - expect(thrown).toBeInstanceOf(CLIError); - expect((thrown as CLIError).exitCode).toBe(5); - // Nothing was created through the symlink, outside --dir. - expect(existsSync(path.join(outside, 'skills'))).toBe(false); - }); - - it('refuses to overwrite a symlinked target file (real disk) with --force — exit 5', async () => { - const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-target-')); - const outsideDir = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-target-')); - const { path: relPath } = renderForTarget('claude', 'testsprite-verify'); - const abs = path.resolve(tmpRoot, relPath); - const nodeFs = await import('node:fs/promises'); - await nodeFs.mkdir(path.dirname(abs), { recursive: true }); - // SKILL.md is a real symlink to a file outside the project root. - const outsideFile = path.join(outsideDir, 'secret.txt'); - await nodeFs.writeFile(outsideFile, 'SECRET', 'utf8'); - symlinkSync(outsideFile, abs, 'file'); - const { deps } = makeCapture(); + // `fs.symlinkSync` needs elevated privileges or Developer Mode on Windows + // (EPERM otherwise) — not guaranteed on hosted CI runners. The underlying + // guard (`inspectTargetPath` fail-closing via `lstat`) is exercised on + // POSIX runners; TODO(DEV-356): revisit if/when a reliable Windows + // symlink-creation path (junctions, or an elevated runner) is available. + it.skipIf(process.platform === 'win32')( + 'refuses to write through a symlinked parent dir (real disk) — exit 5', + async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-parent-')); + const outside = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-')); + // `.claude` is a real symlink to a directory outside the project root. + symlinkSync(outside, path.join(tmpRoot, '.claude'), 'dir'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + skills: ['testsprite-verify'], + force: false, + dir: tmpRoot, + }, + { ...deps }, + ); + } catch (err) { + thrown = err; + } - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: ['claude'], - skills: ['testsprite-verify'], - force: true, - dir: tmpRoot, - }, - { ...deps }, - ); - } catch (err) { - thrown = err; - } + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + // Nothing was created through the symlink, outside --dir. + expect(existsSync(path.join(outside, 'skills'))).toBe(false); + }, + ); + + // Same Windows symlink-privilege caveat as the test above. + it.skipIf(process.platform === 'win32')( + 'refuses to overwrite a symlinked target file (real disk) with --force — exit 5', + async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-target-')); + const outsideDir = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-target-')); + const { path: relPath } = renderForTarget('claude', 'testsprite-verify'); + const abs = path.resolve(tmpRoot, relPath); + const nodeFs = await import('node:fs/promises'); + await nodeFs.mkdir(path.dirname(abs), { recursive: true }); + // SKILL.md is a real symlink to a file outside the project root. + const outsideFile = path.join(outsideDir, 'secret.txt'); + await nodeFs.writeFile(outsideFile, 'SECRET', 'utf8'); + symlinkSync(outsideFile, abs, 'file'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + skills: ['testsprite-verify'], + force: true, + dir: tmpRoot, + }, + { ...deps }, + ); + } catch (err) { + thrown = err; + } - expect(thrown).toBeInstanceOf(CLIError); - expect((thrown as CLIError).exitCode).toBe(5); - // The outside file was NOT overwritten (nor clobbered via the .bak path). - expect(readFileSync(outsideFile, 'utf8')).toBe('SECRET'); - }); + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + // The outside file was NOT overwritten (nor clobbered via the .bak path). + expect(readFileSync(outsideFile, 'utf8')).toBe('SECRET'); + }, + ); }); // --------------------------------------------------------------------------- @@ -2036,12 +2048,12 @@ describe('[P3 round-2] codex --dry-run: composed-size precision + read-failure s const { capture, deps } = makeCapture(); const agentsAbs = path.resolve(CWD, TARGETS.codex.path); - // Old managed section with a 6 KiB body + 26 KiB of user prose. - // existing (~32.9 KiB) + new section (~4.8 KiB) > 32 KiB → the OLD formula - // would warn; the composed replace result (26 KiB + new section) is - // comfortably under budget → no warn expected. + // Old managed section with a 6 KiB body + 25 KiB of user prose. + // existing (~31.9 KiB) + new section (~6.2 KiB: verify codex body + + // onboard aggregate) > 32 KiB → the OLD formula would warn; the composed + // replace result (25 KiB + new section) is under budget → no warn expected. const oldSection = `${MANAGED_SECTION_BEGIN}\n${'o'.repeat(6 * 1024)}\n${MANAGED_SECTION_END}\n`; - const userProse = `# My own AGENTS.md\n${'u'.repeat(26 * 1024)}\n`; + const userProse = `# My own AGENTS.md\n${'u'.repeat(25 * 1024)}\n`; seedFile(agentsAbs, `${userProse}\n${oldSection}`); await runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }); diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index c0b4e00..6f8ce94 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -850,6 +850,63 @@ describe('runWhoami', () => { expect(printed).toEqual(sampleMe); }); + it('renders routing: v3 and the gap advisory when v3Enabled is true', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV3 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV3) }, + ); + expect(capture.stdout.join('\n')).toContain('routing: v3'); + expect(capture.stderr.join('\n')).toContain('[advisory]'); + expect(capture.stderr.join('\n')).toContain('test cancel'); + }); + + it('renders routing: v2 and NO advisory when v3Enabled is false', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV2 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: false }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV2) }, + ); + expect(capture.stdout.join('\n')).toContain('routing: v2'); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('omits the routing line when the backend does not return v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ); + expect(capture.stdout.join('\n')).not.toContain('routing:'); + }); + + it('does not emit the advisory in JSON mode even when v3Enabled is true', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV3 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV3) }, + ); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + const parsed = JSON.parse(capture.stdout.join('')) as MeResponse; + expect(parsed.v3Enabled).toBe(true); + }); + it('dry-run: whitespace-only TESTSPRITE_API_URL falls through to prod default endpoint', async () => { const { capture, deps } = makeCapture(); await runWhoami( diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 59cd43f..e594066 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -22,6 +22,7 @@ import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptSecret } from '../lib/prompt.js'; +import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; export interface MeResponse { userId: string; @@ -37,6 +38,11 @@ export interface MeResponse { email?: string; /** Human-readable display name for the bound account. Absent-safe (dogfood L1866). */ displayName?: string; + /** + * Authoritative per-user V3 routing bit. Absent-safe: older backends omit it, + * so it is only rendered when present. + */ + v3Enabled?: boolean; } export interface AuthDeps { @@ -201,6 +207,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promise { const out = makeOutput(opts.output, deps); const env = deps.env ?? process.env; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); // Resolve the endpoint URL so it can be surfaced in text output. // Dry-run uses the flag/env/default chain without touching credentials. @@ -244,6 +251,8 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi `endpoint: ${resolvedEndpoint}`, `env: ${m.env}`, `scopes: ${m.scopes.join(', ')}`, + // Authoritative routing mode, rendered only when the backend supplies it. + ...(m.v3Enabled !== undefined ? [`routing: ${routingLabel(m.v3Enabled)}`] : []), ]; // C2: warn in text mode when key cannot write/run const missingScopes = (['write:tests', 'run:tests'] as const).filter( @@ -256,6 +265,11 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi } return lines.join('\n'); }); + // When V3 routing is on, warn (text mode only) about the still-open behavior + // gaps. JSON consumers read `v3Enabled` directly; stdout stays pure. + if (opts.output !== 'json' && me.v3Enabled === true) { + emitV3RoutingAdvisory(stderr); + } return me; } diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index 230398a..7a508dd 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -78,6 +78,51 @@ describe('runDoctor — healthy environment', () => { expect(out).toContain('reached GET /me'); }); + it('adds a Routing check (v3) and the gap advisory when /me reports v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { + ...healthyDeps(credentialsPath, { + fetchImpl: makeFetch({ ...OK_ME, v3Enabled: true }), + }), + ...deps, + }, + ); + expect(report.failures).toBe(0); + expect(report.checks.some(c => c.name === 'Routing' && c.detail.includes('v3'))).toBe(true); + expect(capture.stderr.join('\n')).toContain('[advisory]'); + expect(capture.stderr.join('\n')).toContain('test cancel'); + }); + + it('shows Routing v2 and no advisory when v3Enabled is false', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { + ...healthyDeps(credentialsPath, { + fetchImpl: makeFetch({ ...OK_ME, v3Enabled: false }), + }), + ...deps, + }, + ); + expect(report.checks.some(c => c.name === 'Routing' && c.detail.includes('v2'))).toBe(true); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('omits the Routing check when /me does not report v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, // OK_ME has no v3Enabled + ); + expect(report.checks.some(c => c.name === 'Routing')).toBe(false); + expect(report.warnings).toBe(0); + }); + it('never prints the API key anywhere in the report', async () => { writeProfile('default', { apiKey: 'sk-super-secret-value' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 2fa0c57..cceeb13 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -26,6 +26,7 @@ import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; +import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; import { VERSION } from '../version.js'; import { MIN_SUPPORTED_NODE_MAJOR, shouldRejectNodeVersion } from '../version-guard.js'; @@ -49,6 +50,7 @@ export interface DoctorReport { interface MeIdentity { userId?: string; keyId?: string; + v3Enabled?: boolean; } export interface DoctorDeps { @@ -81,6 +83,12 @@ export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Pro }); const endpointCheck = checkEndpoint(config.apiUrl); const hasKey = Boolean(config.apiKey); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + const connectivity = await checkConnectivity(opts, deps, { + hasKey, + endpointOk: endpointCheck.status === 'ok', + }); const checks: DoctorCheck[] = [ { name: 'CLI version', status: 'ok', detail: VERSION }, @@ -88,19 +96,34 @@ export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Pro { name: 'Profile', status: 'ok', detail: config.profile }, endpointCheck, checkCredentials(hasKey, config.profile, opts.dryRun ?? false), - await checkConnectivity(opts, deps, { - hasKey, - endpointOk: endpointCheck.status === 'ok', - }), - checkSkill(cwd, deps), + connectivity.check, ]; + // Informational routing line, only when the backend reported it (no new call). + if (connectivity.v3Enabled !== undefined) { + const label = routingLabel(connectivity.v3Enabled); + checks.push({ + name: 'Routing', + status: 'ok', + detail: + connectivity.v3Enabled === true + ? `${label} (V3 execution routing is ON)` + : `${label} (default routing)`, + }); + } + + checks.push(checkSkill(cwd, deps)); + const failures = checks.filter(check => check.status === 'fail').length; const warnings = checks.filter(check => check.status === 'warn').length; const report: DoctorReport = { checks, failures, warnings }; out.print(report, () => renderDoctor(report)); + if (connectivity.v3Enabled === true) { + emitV3RoutingAdvisory(stderr); + } + if (failures > 0) { // Non-zero exit so `testsprite doctor && ...` gates a CI step or an agent // preflight. The full report already printed above; this line is the stderr @@ -175,11 +198,13 @@ async function checkConnectivity( opts: CommonOptions, deps: DoctorDeps, ctx: { hasKey: boolean; endpointOk: boolean }, -): Promise { +): Promise<{ check: DoctorCheck; v3Enabled?: boolean }> { const name = 'Connectivity'; - if (opts.dryRun) return { name, status: 'warn', detail: 'skipped under --dry-run' }; - if (!ctx.hasKey) return { name, status: 'warn', detail: 'skipped; no API key to test with' }; - if (!ctx.endpointOk) return { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' }; + if (opts.dryRun) return { check: { name, status: 'warn', detail: 'skipped under --dry-run' } }; + if (!ctx.hasKey) + return { check: { name, status: 'warn', detail: 'skipped; no API key to test with' } }; + if (!ctx.endpointOk) + return { check: { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' } }; try { const client = makeHttpClient(opts, { @@ -190,7 +215,10 @@ async function checkConnectivity( }); const me = await client.get('/me'); const who = me.userId ? ` (userId ${me.userId})` : ''; - return { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }; + return { + check: { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }, + v3Enabled: me.v3Enabled, + }; } catch (error) { if (error instanceof ApiError) { if ( @@ -198,14 +226,16 @@ async function checkConnectivity( error.code === 'AUTH_INVALID' || error.code === 'AUTH_FORBIDDEN' ) { - return { name, status: 'fail', detail: `API key rejected (${error.code})` }; + return { check: { name, status: 'fail', detail: `API key rejected (${error.code})` } }; } - return { name, status: 'fail', detail: `GET /me failed (${error.code})` }; + return { check: { name, status: 'fail', detail: `GET /me failed (${error.code})` } }; } return { - name, - status: 'fail', - detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, + check: { + name, + status: 'fail', + detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, + }, }; } } diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index daea851..0f8c51c 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -6,11 +6,13 @@ import { ApiError } from '../lib/errors.js'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; import { type CliProject, + type CliDeleteProjectResponse, type CliUpdateProjectResponse, createProjectCommand, runAutoAuth, runCreate, runCredential, + runDelete, runGet, runList, runUpdate, @@ -74,10 +76,10 @@ describe('createProjectCommand', () => { errorSpy.mockRestore(); }); - it('exposes list, get, create, update, credential and auto-auth subcommands', () => { + it('exposes list, get, create, update, delete, credential and auto-auth subcommands', () => { const project = createProjectCommand(); const names = project.commands.map(c => c.name()).sort(); - expect(names).toEqual(['auto-auth', 'create', 'credential', 'get', 'list', 'update']); + expect(names).toEqual(['auto-auth', 'create', 'credential', 'delete', 'get', 'list', 'update']); }); it('list exposes the pagination flags from the design contract', () => { @@ -681,6 +683,34 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + + it('rejects --description with VALIDATION_ERROR (exit 5), no network — projects have no description', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network — validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Desc Project', + targetUrl: 'https://example.com', + description: 'a human description', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); }); // --------------------------------------------------------------------------- @@ -946,6 +976,140 @@ describe('runUpdate', () => { }); }); +describe('runDelete', () => { + it('refuses without --confirm and never hits the network (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: {} }; + }); + await expect( + runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'confirm' }), + }); + expect(called).toBe(0); + }); + + it('DELETEs /projects/{id} with a minted idempotency-key when --confirm is set', async () => { + const { credentialsPath } = makeCreds(); + const deleteResponse: CliDeleteProjectResponse = { + projectId: 'proj_alpha', + deletedAt: '2026-05-16T10:00:00.000Z', + }; + let seenUrl = ''; + let seenMethod = ''; + let seenIdemKey: string | null = null; + const fetchImpl = (async (input: Parameters[0], init: RequestInit = {}) => { + seenUrl = typeof input === 'string' ? input : (input as { url: string }).url; + seenMethod = init.method ?? 'GET'; + seenIdemKey = new Headers(init.headers).get('idempotency-key'); + return new Response(JSON.stringify(deleteResponse), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + + const result = await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: true, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(seenMethod).toBe('DELETE'); + expect(seenUrl).toContain('/api/cli/v1/projects/proj_alpha'); + expect(seenIdemKey).toMatch(/^cli-delete-[0-9a-f-]{36}$/); + expect(result.projectId).toBe('proj_alpha'); + expect(result.deletedAt).toBe('2026-05-16T10:00:00.000Z'); + }); + + it('forwards a caller-supplied --idempotency-key verbatim', async () => { + const { credentialsPath } = makeCreds(); + let seenIdemKey: string | null = null; + const fetchImpl = (async (_input: Parameters[0], init: RequestInit = {}) => { + seenIdemKey = new Headers(init.headers).get('idempotency-key'); + return new Response( + JSON.stringify({ projectId: 'proj_alpha', deletedAt: '2026-05-16T10:00:00.000Z' }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof fetch; + + await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: true, + idempotencyKey: 'idem-del-001', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(seenIdemKey).toBe('idem-del-001'); + }); + + it('--dry-run bypasses --confirm and returns the canned sample without network', async () => { + resetDryRunBannerForTesting(); + const { credentialsPath } = makeCreds(); + const err: string[] = []; + // No fetchImpl → the client-factory dry-run fetch serves the samples.ts value. + const result = await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_b3c91efa', + confirm: false, + }, + { credentialsPath, stdout: () => {}, stderr: line => err.push(line) }, + ); + + expect(result.projectId).toBe('project_b3c91efa'); + expect(result.deletedAt).toBe('2026-05-16T00:00:00.000Z'); + expect(err).toContain(DRY_RUN_BANNER); + }); + + it('renders text mode with projectId and deletedAt', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { projectId: 'proj_text', deletedAt: '2026-05-16T10:00:00.000Z' }, + })); + const out: string[] = []; + await runDelete( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_text', + confirm: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, + ); + const block = out.join('\n'); + expect(block).toContain('projectId proj_text'); + expect(block).toContain('deletedAt 2026-05-16T10:00:00.000Z'); + }); +}); + describe('runCredential', () => { interface Captured { url: string; diff --git a/src/commands/project.ts b/src/commands/project.ts index 96f2fc8..0a843e5 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -110,7 +110,8 @@ export interface CliCreateProjectRequest { type: 'frontend' | 'backend'; name: string; targetUrl?: string; - description?: string; + // `description` is intentionally not part of the wire request — projects have + // no description field. The `--description` flag is rejected client-side. username?: string; password?: string; instruction?: string; @@ -154,8 +155,13 @@ export async function runCreate( if (opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } - if (opts.description !== undefined && opts.description.length > 2000) { - throw localValidationError('--description must be at most 2000 characters'); + // `--description` is not supported on projects — no project entity stores a + // description, and the backend rejects it with a 422. Fail fast client-side + // with an actionable message instead of a wasted round trip. + if (opts.description !== undefined) { + throw localValidationError( + '--description is not supported for projects; omit it (test-level descriptions are set on `test create`)', + ); } // P2-7: guard --url against localhost/RFC1918/non-http(s) (same rules as @@ -210,7 +216,6 @@ export async function runCreate( type: opts.type, name: opts.name, ...(opts.targetUrl !== undefined ? { targetUrl: opts.targetUrl } : {}), - ...(opts.description !== undefined ? { description: opts.description } : {}), ...(opts.username !== undefined ? { username: opts.username } : {}), ...(password !== undefined ? { password } : {}), ...(opts.instruction !== undefined ? { instruction: opts.instruction } : {}), @@ -346,6 +351,80 @@ export async function runUpdate( return updated; } +// --------------------------------------------------------------------------- +// project delete +// --------------------------------------------------------------------------- + +export interface CliDeleteProjectResponse { + projectId: string; + deletedAt: string; +} + +interface DeleteOptions extends CommonOptions { + projectId: string; + /** Hard gate — required (unless `--dry-run` is set). No interactive prompts. */ + confirm: boolean; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; +} + +/** + * `project delete --confirm` — permanent cascade delete via + * DELETE /projects/{id}. + * + * The server deletes the project together with everything under it — its + * frontend/backend sub-projects, all their tests, and backend fixtures — + * matching the Portal's own delete behavior. There is no restore window. + * + * **`--confirm` is required** (unless `--dry-run`). Without either, the CLI + * exits 5 `VALIDATION_ERROR` with a typed envelope explaining the convention. + * The CLI never prompts interactively (CI-friendly contract). Re-delete on an already-deleted (or missing) project returns 404 from + * the server; the CLI surfaces the envelope as-is (exit 4), no client branching. + */ +export async function runDelete( + opts: DeleteOptions, + deps: ProjectDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + if (opts.projectId === undefined || opts.projectId.trim().length === 0) { + throw localValidationError(' is required'); + } + + if (!opts.confirm && !opts.dryRun) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Refusing to delete without --confirm.', + nextAction: + 'This permanently deletes the project and everything under it — its ' + + 'sub-projects, all their tests, and backend fixtures (no restore window). ' + + 'The CLI convention is explicit confirmation for destructive operations. ' + + 'Re-run with --confirm. (--dry-run also works without --confirm.)', + requestId: 'local', + details: { field: 'confirm', reason: 'required for destructive operation' }, + }, + }); + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-delete-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + const response = await client.delete( + `/projects/${encodeURIComponent(opts.projectId)}`, + { + headers: { 'idempotency-key': idempotencyKey }, + }, + ); + + out.print(response, data => renderDeleteText(data as CliDeleteProjectResponse)); + return response; +} + // --------------------------------------------------------------------------- // project credential — set the static backend credential // --------------------------------------------------------------------------- @@ -616,7 +695,10 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--type ', 'project type (required)') .option('--name ', 'project name (required)') .option('--url ', 'target URL (required for frontend)') - .option('--description ', 'optional human description') + .option( + '--description ', + 'not supported — projects have no description (test-level descriptions are set on `test create`)', + ) .option('--username ', 'optional auth username') .option('--password ', 'optional auth password (use --password-file for non-interactive)') .option('--password-file ', 'read password from file instead of inline flag') @@ -684,6 +766,35 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { ); }); + project + .command('delete ') + .description( + 'Permanently delete a project and everything under it (sub-projects,\n' + + 'their tests, and backend fixtures). Requires --confirm.\n' + + '\nExit codes:\n' + + ' 0 success\n' + + ' 3 auth error\n' + + ' 4 project not found (or already deleted)\n' + + ' 5 validation error (e.g., missing --confirm)', + ) + .option('--confirm', 'required: explicit confirmation for the destructive operation', false) + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: DeleteFlagOpts, command: Command) => { + await runDelete( + { + ...resolveCommonOptions(command), + projectId, + confirm: cmdOpts.confirm === true, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + project .command('credential ') .description( @@ -808,6 +919,11 @@ interface UpdateFlagOpts { idempotencyKey?: string; } +interface DeleteFlagOpts { + confirm?: boolean; + idempotencyKey?: string; +} + interface CredentialFlagOpts { type: string; credential?: string; @@ -952,6 +1068,10 @@ function renderUpdateText(r: CliUpdateProjectResponse): string { ].join('\n'); } +function renderDeleteText(r: CliDeleteProjectResponse): string { + return [`projectId ${r.projectId}`, `deletedAt ${r.deletedAt}`].join('\n'); +} + function localValidationError(message: string): ApiError { return ApiError.fromEnvelope({ error: { diff --git a/src/commands/test.cancel.spec.ts b/src/commands/test.cancel.spec.ts new file mode 100644 index 0000000..72496b0 --- /dev/null +++ b/src/commands/test.cancel.spec.ts @@ -0,0 +1,309 @@ +/** + * Unit tests for `test cancel `. + * + * Covers: single-id happy path (text + json), `alreadyCancelled` advisory, + * multi-id mixed summary + exit precedence, 404, 409, dry-run. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; +import { ApiError } from '../lib/errors.js'; +import type { CancelRunResponse } from '../lib/runs.types.js'; +import { runTestCancel, type CliCancelSummary } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13503', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-cancel-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +function makeCancelResponse( + runId: string, + overrides: Partial = {}, +): CancelRunResponse { + return { + runId, + testId: 'test_xyz', + projectId: 'project_1', + userId: 'user_1', + status: 'cancelled', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 2, passedCount: 2, failedCount: 0 }, + alreadyCancelled: false, + ...overrides, + }; +} + +function errorBody(code: string, details: Record = {}) { + const statusMap: Record = { + NOT_FOUND: 404, + CONFLICT: 409, + AUTH_FORBIDDEN: 403, + }; + return { + status: statusMap[code] ?? 400, + body: { + error: { + code, + message: `Error: ${code}`, + nextAction: 'do something', + requestId: 'req_test', + details, + }, + }, + }; +} + +/** Extract the runId embedded in a `POST /runs/{runId}/cancel` URL. */ +function runIdFromCancelUrl(url: string): string | undefined { + const match = /\/runs\/([^/]+)\/cancel/.exec(url); + return match?.[1]; +} + +// --------------------------------------------------------------------------- +// Single id — happy path +// --------------------------------------------------------------------------- + +describe('runTestCancel — single id happy path', () => { + it('CXL-1: fresh cancel of a queued/running run → 200, alreadyCancelled:false, no advisory', async () => { + const { credentialsPath } = makeCreds(); + let seenUrl = ''; + let seenMethod = ''; + const fetchImpl = makeFetch((url, init) => { + seenUrl = url; + seenMethod = init.method ?? 'GET'; + return { body: makeCancelResponse('run_abc') }; + }); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + + expect(seenMethod).toBe('POST'); + expect(runIdFromCancelUrl(seenUrl)).toBe('run_abc'); + expect(result.status).toBe('cancelled'); + expect(result.alreadyCancelled).toBe(false); + expect(stderrLines.some(l => l.includes('already cancelled'))).toBe(false); + }); + + it('renders the run card in text mode with status cancelled', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeCancelResponse('run_abc') })); + const stdoutLines: string[] = []; + await runTestCancel( + { profile: 'default', output: 'text', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => {} }, + ); + const block = stdoutLines.join('\n'); + expect(block).toContain('run_abc'); + expect(block).toContain('status'); + expect(block).toContain('cancelled'); + }); + + it('CXL-5: alreadyCancelled:true → [advisory] stderr line, still exit 0 (no throw)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: makeCancelResponse('run_abc', { alreadyCancelled: true }), + })); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + expect(result.alreadyCancelled).toBe(true); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('already cancelled'))).toBe( + true, + ); + }); + + it('CXL-6: unknown/cross-tenant runId → 404 propagates as ApiError exit 4', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('NOT_FOUND')); + const err = await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_ghost'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(4); + }); + + it('CXL-4: already-terminal run → 409 propagates as ApiError exit 6', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('CONFLICT', { status: 'passed' })); + const err = await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_done'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(6); + }); +}); + +// --------------------------------------------------------------------------- +// Multi-id — summary + exit precedence (CXL-11) +// --------------------------------------------------------------------------- + +describe('runTestCancel — multi-id summary + exit precedence (CXL-11)', () => { + it('all cancelled/alreadyCancelled → exit 0, summary buckets correct', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_2') { + return { body: makeCancelResponse(runId, { alreadyCancelled: true }) }; + } + return { body: makeCancelResponse(runId) }; + }); + const result = (await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_1', 'run_2'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + )) as CliCancelSummary; + expect(result.cancelled).toEqual(['run_1']); + expect(result.alreadyCancelled).toEqual(['run_2']); + expect(result.conflicts).toEqual([]); + expect(result.notFound).toEqual([]); + // Stable machine shape (codex finding 2): errors is ALWAYS present, + // an empty array on full success — never an absent key. + expect(result.errors).toEqual([]); + }); + + it('mixed: cancelled + conflict + notFound → notFound wins (exit 4), all buckets populated', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'failed' }); + if (runId === 'run_ghost') return errorBody('NOT_FOUND'); + return { body: makeCancelResponse(runId) }; + }); + const err = await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict', 'run_ghost'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err.exitCode).toBe(4); // notFound outranks conflict + expect(err.message).toContain('run_ghost'); + }); + + it('cancelled + conflict, no notFound → conflict wins (exit 6)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'blocked' }); + return { body: makeCancelResponse(runId) }; + }); + const err = await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err.exitCode).toBe(6); + expect(err.message).toContain('run_conflict'); + expect(err.message).toContain('blocked'); + }); + + it('JSON summary shape carries the conflicting run status', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'passed' }); + return { body: makeCancelResponse(runId) }; + }); + const stdoutLines: string[] = []; + await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict'], + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + }, + ).catch(() => {}); + const summary = JSON.parse(stdoutLines.join('\n')) as CliCancelSummary; + expect(summary.cancelled).toEqual(['run_ok']); + expect(summary.conflicts).toEqual([{ runId: 'run_conflict', status: 'passed' }]); + }); +}); + +// --------------------------------------------------------------------------- +// Dry-run +// --------------------------------------------------------------------------- + +describe('runTestCancel — dry-run', () => { + it('single id: prints the dry-run banner and a canned cancelled envelope, no real network', async () => { + resetDryRunBannerForTesting(); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: true, runIds: ['run_dry'] }, + { stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + expect(stderrLines.some(l => l.includes(DRY_RUN_BANNER))).toBe(true); + expect(result.status).toBe('cancelled'); + expect(result.alreadyCancelled).toBe(false); + }); +}); diff --git a/src/commands/test.quickwins.spec.ts b/src/commands/test.quickwins.spec.ts index c781300..322000c 100644 --- a/src/commands/test.quickwins.spec.ts +++ b/src/commands/test.quickwins.spec.ts @@ -937,3 +937,113 @@ describe('runDeleteBatch (dogfood L1796)', () => { expect(flagNames).toContain('--status'); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 (codex finding 2) — create-batch --run --wait interrupt partial +// carries the dispatched runIds, not empty placeholders +// --------------------------------------------------------------------------- + +describe('create-batch --run --wait — InterruptError partial names dispatched runIds (DEV-331)', () => { + it('interrupt mid-poll → partial rows carry the runIds recorded at trigger time', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-dev331-cbrun-')); + for (let i = 0; i < 2; i++) { + writeFileSync( + join(dir, `plan_${i}.json`), + JSON.stringify({ ...PLAN_SPEC, name: `Plan ${i}` }), + 'utf8', + ); + } + + const { ShutdownController } = await import('../lib/interrupt.js'); + const { InterruptError } = await import('../lib/errors.js'); + const shutdown = new ShutdownController(); + + // batch create resolves; each trigger POST resolves with a per-test runId; + // every run poll hangs until the composed signal aborts. + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/batch') && init.method === 'POST') { + return new Response(JSON.stringify(batchCreateResp(2)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (init.method === 'POST' && /\/tests\/[^/]+\/runs$/.test(url)) { + const testId = /\/tests\/([^/]+)\/runs$/.exec(url)![1]!; + return new Response( + JSON.stringify({ + runId: `run_${testId}`, + status: 'queued', + enqueuedAt: '2026-07-09T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + } + // GET /runs/{id} long-poll: hang until aborted. + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const pending = runCreateBatch( + { + plans: '', + planFromDir: dir, + run: true, + wait: true, + timeoutSeconds: 600, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + fetchImpl, + stdout: (l: string) => stdoutLines.push(l), + stderr: (l: string) => stderrLines.push(l), + sleep: () => Promise.resolve(), + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 25); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + + // The partial must name the real runIds recorded at trigger time — + // members mid-poll have no settled result, but their runId is known. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + results: Array<{ testId: string; runId: string; status: string }>; + }; + const byTestId = new Map(stdoutJson.results.map(r => [r.testId, r])); + expect(byTestId.get('test_batch_0')?.runId).toBe('run_test_batch_0'); + expect(byTestId.get('test_batch_0')?.status).toBe('running'); + expect(byTestId.get('test_batch_1')?.runId).toBe('run_test_batch_1'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('run_test_batch_0'); + expect(stderrBlock).toContain('run_test_batch_1'); + }); +}); diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 9f16c4b..3797f14 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -9,7 +9,8 @@ import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; import type { FetchImpl } from '../lib/http.js'; import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; @@ -4881,3 +4882,98 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach during batch rerun --wait (SIG-6) +// --------------------------------------------------------------------------- + +describe('R-BAT: batch rerun --wait — InterruptError partial lists all dispatched runIds (DEV-331)', () => { + it('interrupt mid fan-out → stdout partial covers every accepted runId, honest stderr, exit 130', async () => { + const creds = makeCreds(); + const shutdown = new ShutdownController(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + // Batch trigger resolves; every run poll hangs until the composed signal aborts. + const fetchImpl: FetchImpl = (async (input: unknown, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/batch/rerun')) { + return new Response(JSON.stringify(batchResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as FetchImpl; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const pending = runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 10); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + + // SIG-6: the partial lists ALL dispatched runIds, marked running. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ runId: string; status: string }>; + }; + const byRunId = new Map(stdoutJson.accepted.map(r => [r.runId, r.status])); + expect(byRunId.get('run_b1')).toBe('running'); + expect(byRunId.get('run_b2')).toBe('running'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('run_b1'); + expect(stderrBlock).toContain('run_b2'); + }); +}); diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index c0ff083..51b527f 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -10,7 +10,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; import type { FetchImpl } from '../lib/http.js'; import type { RunResponse, TriggerRunResponse, BatchRunFreshResponse } from '../lib/runs.types.js'; @@ -1699,6 +1700,88 @@ describe('C2 — backend run renders steps: n/a (backend) in text mode', () => { expect(out).toContain('steps n/a (backend)'); expect(out).not.toContain('0/0'); }); + + it('standalone BE run --wait: text probes /tests/{id} → n/a (backend); JSON never probes (DEV-282)', async () => { + // Standalone `test run ` supplies NO type hint, and the run row is + // terminal on the first poll (BE rows finalize server-side now), so + // `beFallbackUsed` stays false. In TEXT mode the card must still read + // `n/a (backend)`, resolved via a one-time `GET /tests/{id}` probe; in JSON + // mode the output is already correct and must NOT pay for that round-trip. + const { credentialsPath } = makeCreds(); + const passedBeRun: RunResponse = { + runId: 'run_282', + testId: 'be_282', + projectId: 'p1', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:02.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 0, completed: 0, passedCount: 0, failedCount: 0 }, + }; + const makeHandler = (urls: string[]) => (url: string) => { + urls.push(url); + if (url.includes('/tests/be_282/runs')) { + return { + body: { + runId: 'run_282', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + if (url.includes('/runs/run_282')) return { body: passedBeRun }; + // Bare type probe: GET /tests/be_282 (no /runs, no /result suffix). + if (/\/tests\/be_282$/.test(url)) return { body: { id: 'be_282', type: 'backend' } }; + return { status: 404, body: {} }; + }; + const runOnce = async (output: 'text' | 'json') => { + const urls: string[] = []; + const stdoutLines: string[] = []; + await runTestRun( + { + profile: 'default', + output, + debug: false, + dryRun: false, + testId: 'be_282', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(makeHandler(urls)), + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + return { urls, out: stdoutLines.join('\n') }; + }; + + // Text mode: probes the type and renders the honest backend placeholder. + const text = await runOnce('text'); + expect(text.out).toContain('steps n/a (backend)'); + expect(text.out).not.toContain('0/0'); + expect(text.urls.some(u => /\/tests\/be_282$/.test(u))).toBe(true); + + // JSON mode: no extra probe; the wire envelope ships stepSummary verbatim. + const json = await runOnce('json'); + expect(json.urls.some(u => /\/tests\/be_282$/.test(u))).toBe(false); + const parsed = JSON.parse(json.out); + expect(parsed.status).toBe('passed'); + expect(parsed.stepSummary).toEqual({ total: 0, completed: 0, passedCount: 0, failedCount: 0 }); + }); }); // --------------------------------------------------------------------------- @@ -3748,3 +3831,80 @@ describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out p expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach on SIGINT during test run --wait +// --------------------------------------------------------------------------- + +describe('runTestRun --wait — InterruptError graceful detach (DEV-331)', () => { + it('SIG-1: trigger succeeds, poll interrupted → partial to stdout + honest stderr + exit 130', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + // Trigger POST resolves; the subsequent GET /runs/{id} long-poll hangs + // until the composed signal aborts (real-fetch contract). + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (init.method === 'POST') { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + void url; + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + + const pending = runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 5); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('testsprite test wait run_abc'); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index f7c7602..9493908 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -120,6 +120,7 @@ describe('createTestCommand — surface', () => { const names = test.commands.map(c => c.name()).sort(); expect(names).toEqual([ 'artifact', + 'cancel', 'code', 'create', 'create-batch', @@ -834,6 +835,40 @@ describe('runGet', () => { expect(out.join('\n')).not.toContain('planSteps:'); }); + it('renders produces/consumes/category when the facade ships them', async () => { + const withDeps: CliTest = { + ...FE_TEST, + produces: ['user_id', 'order_id'], + consumes: ['session_token'], + category: 'teardown', + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: withDeps })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('produces: user_id, order_id'); + expect(block).toContain('consumes: session_token'); + expect(block).toContain('category: teardown'); + }); + + it('omits produces/consumes/category lines when absent', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: FE_TEST })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).not.toContain('produces:'); + expect(block).not.toContain('consumes:'); + expect(block).not.toContain('category:'); + }); + it('NOT_FOUND envelope from server propagates as ApiError exit 4', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ @@ -6275,6 +6310,74 @@ describe('runUpdate', () => { expect(seenBody).toEqual({ priority: 'p2' }); }); + it('threads --produces/--needs/--category into the PUT body with wire names', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + produces: ['user_id', 'order_id'], + needs: ['session_token'], + category: 'teardown', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ + produces: ['user_id', 'order_id'], + consumes: ['session_token'], + category: 'teardown', + }); + }); + + it('accepts a dependency-only update (not rejected as a no-op)', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + category: 'teardown', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ category: 'teardown' }); + }); + + it('omits empty dependency arrays from the body', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: 'renamed', + produces: [], + needs: [], + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ name: 'renamed' }); + }); + it('respects a caller-supplied --idempotency-key', async () => { const { credentialsPath } = makeCreds(); let seenKey: string | null = null; diff --git a/src/commands/test.ts b/src/commands/test.ts index 0602afc..837c170 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -38,10 +38,12 @@ import { import { ApiError, CLIError, + InterruptError, RequestTimeoutError, TransportError, localValidationError, } from '../lib/errors.js'; +import { globalShutdown, type ShutdownHandle } from '../lib/interrupt.js'; import { assertIdempotencyKey, requireArrayLength, @@ -75,6 +77,7 @@ import type { RunSource, BatchRunFreshResponse, BatchRunFreshAccepted, + CancelRunResponse, } from '../lib/runs.types.js'; import { RUN_SOURCES } from '../lib/runs.types.js'; import { assertNotLocal } from '../lib/target-url.js'; @@ -153,6 +156,14 @@ export interface CliTest { * no priority has been set. Text mode surfaces it only when truthy. */ priority?: string | null; + /** + * Backend-only dependency declarations that drive wave ordering. + * Optional on the wire so older facades that don't ship them still + * type-check; text mode surfaces them only when present/non-empty. + */ + produces?: string[] | null; + consumes?: string[] | null; + category?: string | null; } export type CliPublicStatus = @@ -292,6 +303,18 @@ export interface CliLatestResult { * former `{passed,failed,skipped}` count object). */ summary: string; + /** + * Captured stdout (`api_output`) from a backend-test execution. + * Present (possibly null) only for backend tests; omitted for FE/MCP and on + * older backends that omit them. Capped at 50 KB UTF-8 by the server. + */ + apiOutput?: string | null; + /** + * Python traceback from a backend-test execution (ends with the + * `ExceptionType: message` line). Present (possibly null) only for backend + * tests; omitted for FE/MCP and on older backends. + */ + trace?: string | null; /** * §6.5.1 (M2.1 piece 3) — inline failure analysis. Present when the * caller passed `--include-analysis` (`?includeAnalysis=true` on @@ -382,6 +405,14 @@ export interface CliFailureBlock { */ recommendedFixTarget: CliFixTarget | null; evidence: CliEvidence[]; + /** + * Backend-test execution artifacts, surfaced on the failure block + * so triage has stdout + traceback next to the hypothesis (mirrors + * `result.apiOutput` / `result.trace`). Present (possibly null) only for + * backend failures; omitted for FE and on older backends. + */ + apiOutput?: string | null; + trace?: string | null; } /** §6.7 wire shape — one atomic snapshot of the latest failing run. */ @@ -413,6 +444,36 @@ export interface TestDeps { * no-op in tests to avoid real delays. */ sleep?: (ms: number) => Promise; + /** + * Graceful-detach coordinator for the `--wait` paths (DEV-331 piece 1). + * Defaults to the process-wide `globalShutdown`; tests inject their own + * controller and abort it to simulate Ctrl-C deterministically (same + * pattern as `sleep` / `fetchImpl`). + */ + shutdown?: ShutdownHandle; +} + +/** The effective shutdown handle for a command invocation (DEV-331). */ +function shutdownOf(deps: TestDeps): ShutdownHandle { + return deps.shutdown ?? globalShutdown; +} + +/** + * The honest-detach stderr line (DEV-331 D1 — the heart of the ticket): + * a Ctrl-C detaches the local wait only; the server-side run keeps executing + * AND billing. Names both re-attach (`test wait`) and the real cancel + * (`test cancel`, piece 3) so the user always has a path to actually stop it. + */ +function interruptDetachMessage(err: InterruptError, runIds: string[]): string { + const subject = + runIds.length === 1 + ? `Run ${runIds[0]} is still executing on the server and will keep running (and billing) until it finishes.` + : `${runIds.length} runs are still executing on the server and will keep running (and billing) until they finish.`; + return ( + `Interrupted (${err.signal}). ${subject}\n` + + ` Re-attach with: testsprite test wait ${runIds.join(' ')}\n` + + ` Cancel with: testsprite test cancel ${runIds.join(' ')}` + ); } type CommonOptions = FactoryCommonOptions; @@ -1274,6 +1335,12 @@ interface UpdateOptions extends CommonOptions { description?: string; /** Optional new priority. Enum-validated CLI-side. */ priority?: CliCreatePriority; + /** Backend-only: variable names this test produces (repeatable --produces). */ + produces?: string[]; + /** Backend-only: variable names this test consumes (repeatable --needs); wire field `consumes`. */ + needs?: string[]; + /** Backend-only: free-text wave category (--category). */ + category?: string; /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ idempotencyKey?: string; } @@ -1330,11 +1397,14 @@ export async function runUpdate( const hasName = opts.name !== undefined; const hasDescription = opts.description !== undefined; const hasPriority = opts.priority !== undefined; - if (!hasName && !hasDescription && !hasPriority) { + const hasProduces = opts.produces !== undefined && opts.produces.length > 0; + const hasNeeds = opts.needs !== undefined && opts.needs.length > 0; + const hasCategory = opts.category !== undefined; + if (!hasName && !hasDescription && !hasPriority && !hasProduces && !hasNeeds && !hasCategory) { throw localValidationError( 'fields', - 'at least one of --name / --description / --priority must be set', - ['name', 'description', 'priority'], + 'at least one of --name / --description / --priority / --produces / --needs / --category must be set', + ['name', 'description', 'priority', 'produces', 'needs', 'category'], ); } @@ -1349,10 +1419,13 @@ export async function runUpdate( // is the intended wire shape — but we build the body deliberately // so the contract is auditable rather than dependent on // JSON.stringify undefined-skipping. - const body: Record = {}; + const body: Record = {}; if (hasName) body.name = opts.name!; if (hasDescription) body.description = opts.description!; if (hasPriority) body.priority = opts.priority!; + if (hasProduces) body.produces = opts.produces!; + if (hasNeeds) body.consumes = opts.needs!; + if (hasCategory) body.category = opts.category!; const client = makeClient(opts, deps); const out = makeOutput(opts.output, deps); @@ -2583,7 +2656,14 @@ async function runBatchRun( * * Returns a CliBatchRunResult. Never throws — errors are captured * into the result's `error` field so one failure doesn't abort siblings. + * (Exception: InterruptError rethrows so the collect point can print the + * partial — DEV-331.) */ + // testId → dispatched runId for members still mid-poll; the interrupt + // partial reads it because a member's runId is local to triggerOne until + // the poll settles (DEV-331, codex finding 2). + const dispatchedRunIds = new Map(); + async function triggerOne(testId: string): Promise { // Mint a fresh idempotency key per run — MUST NOT reuse the create key. const runIdempotencyKey = `cli-batch-run-${randomUUID()}`; @@ -2673,6 +2753,10 @@ async function runBatchRun( triggerResponse = result.body; break; // success — exit the outer retry loop } catch (err) { + // Interrupt must reject the fan-out (the collect point prints the + // partial for every spec), never flatten into a per-member outcome + // that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; // RATE_LIMITED outer retry. Since the HTTP layer no longer retries // RATE_LIMITED (retryOnRateLimit: false above), every 429 reaches here // on the first attempt. @@ -2809,6 +2893,12 @@ async function runBatchRun( } } + // Record the dispatched runId (fresh trigger OR conflict-resume) so the + // interrupt partial at the collect point can name every in-flight run — + // a member's runId is otherwise local until its poll settles (DEV-331, + // codex finding 2). + if (triggerResponse.runId) dispatchedRunIds.set(testId, triggerResponse.runId); + if (!opts.wait) { // No-wait path: return the trigger response as-is. if (opts.output !== 'json') { @@ -2850,11 +2940,14 @@ async function runBatchRun( finalRun = await pollRunUntilTerminal(client, triggerResponse.runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[batch-run][verbose] ${testId}: ${msg}`) : undefined, }); } catch (err) { + // Interrupt rejects the fan-out — see the trigger-stage catch above. + if (err instanceof InterruptError) throw err; if (err instanceof TimeoutError) { if (opts.output !== 'json') { stderrFn( @@ -2920,24 +3013,59 @@ async function runBatchRun( let nextIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && nextIdx < testIds.length) { - const testId = testIds[nextIdx++]!; - inFlight++; - triggerOne(testId) - .then(result => { - batchRunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && nextIdx >= testIds.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && nextIdx < testIds.length) { + const testId = testIds[nextIdx++]!; + inFlight++; + triggerOne(testId) + .then(result => { + batchRunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= testIds.length) resolve(); + }) + .catch(reject); + } + } + startNext(); + if (testIds.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): leave stdout parseable — settled members keep + // their real status, unfinished ones are marked running — then rethrow so + // index.ts exits 128+signum. + if (fanOutErr instanceof InterruptError) { + const settled = new Map(batchRunResults.map(r => [r.testId, r] as const)); + // Members mid-poll have no settled result yet — their runId comes from + // the dispatchedRunIds map recorded at trigger time (codex finding 2). + const partialResults = testIds.map( + (testId): CliBatchRunResult => + settled.get(testId) ?? { + testId, + runId: dispatchedRunIds.get(testId) ?? '', + status: dispatchedRunIds.has(testId) ? 'running' : 'not_dispatched', + codeVersion: '', + }, + ); + out.print({ results: partialResults }, () => + partialResults.map(r => `${r.testId} ${r.runId || '-'} ${r.status}`).join('\n'), + ); + const unfinished = partialResults + .filter(r => r.status === 'running' && r.runId) + .map(r => r.runId); + if (unfinished.length > 0) { + stderrFn(interruptDetachMessage(fanOutErr, unfinished)); + } else { + stderrFn( + `Interrupted (${fanOutErr.signal}). Already-triggered runs keep executing (and billing) server-side; ` + + `check them with: testsprite test list`, + ); } } - startNext(); - if (testIds.length === 0) resolve(); - }); + throw fanOutErr; + } // Sort by testId order (same as input order for stable output). batchRunResults.sort((a, b) => testIds.indexOf(a.testId) - testIds.indexOf(b.testId)); @@ -5100,6 +5228,36 @@ function renderRunResponseText( return lines.join('\n'); } +/** + * Best-effort resolve whether a finished run's test is a backend test, for the + * text run-card's step line + failure hint only (DEV-282). Returns true with no + * network call when already known (create-chain `--type`, or the BE wait + * fallback fired). Otherwise, in TEXT mode only, issues one `GET /tests/{id}`; + * any error → false (render the numeric step summary as before). JSON mode + * never probes — its envelope carries `stepSummary` verbatim and has no + * "n/a (backend)" concept, so it is already correct. + * + * This closes the standalone-path gap: `test run ` / `test wait ` + * never supply a type hint, and now that BE run rows finalize server-side + * (backend-v2.0 #551/#555) the wait fallback rarely fires — so a backend run + * card would otherwise show a misleading `steps 0/0 (passed=0, failed=0)`. + */ +async function resolveRunCardIsBackend( + client: ResultReadClient, + testId: string | undefined, + output: string, + alreadyKnown: boolean, +): Promise { + if (alreadyKnown) return true; + if (output === 'json' || !testId) return false; + try { + const test = await client.get(`/tests/${encodeURIComponent(testId)}`); + return test.type === 'backend'; + } catch { + return false; // best-effort — fall back to the numeric step summary. + } +} + /** * Render a `TriggerRunResponse` (no-wait path) to human-readable text. */ @@ -5259,7 +5417,8 @@ export async function runTestRun( stderrFn( `[advisory] Run already in flight (runId: ${currentRunId}, ` + `target: ${inFlightRun.targetUrl}). ` + - `Attaching to that run's --wait poll instead of creating a new one.`, + `Attaching to that run's --wait poll instead of creating a new one. ` + + `To stop it instead: testsprite test cancel ${currentRunId}`, ); triggerResponse = { runId: currentRunId, @@ -5285,12 +5444,16 @@ export async function runTestRun( // Auto-resume but emit a stronger advisory so the caller is aware // they are attaching to the project default. + // SIG-9 (DEV-331 final): the in-flight run is NOT auto-cancelled — + // name the real `test cancel` command instead of the old (false) + // "cancel with Ctrl-C" claim. stderrFn( `[advisory] Run already in flight (runId: ${currentRunId}` + (inFlightTargetUrl ? `, target: ${inFlightTargetUrl}` : '') + `). Auto-resuming wait on in-flight run. ` + - `If you needed a specific target URL, cancel with Ctrl-C and ` + - `re-trigger with --target-url.`, + `If you needed a specific target URL, cancel it with ` + + `testsprite test cancel ${currentRunId}, or re-trigger with ` + + `--target-url after it finishes.`, ); triggerResponse = { runId: currentRunId, @@ -5359,6 +5522,7 @@ export async function runTestRun( finalRun = await pollRunUntilTerminal(client, triggerResponse.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -5390,13 +5554,14 @@ export async function runTestRun( ]; if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); return lines.join('\n'); }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${triggerResponse.runId}.`, - nextAction: `Resume polling: testsprite test wait ${triggerResponse.runId}`, + nextAction: `Resume polling: testsprite test wait ${triggerResponse.runId}, or cancel it: testsprite test cancel ${triggerResponse.runId}`, requestId: 'local', details: { runId: triggerResponse.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -5423,14 +5588,39 @@ export async function runTestRun( const lines = [`runId ${p.runId}`, `status ${p.status} (request timed out)`]; if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); return lines.join('\n'); }); stderrFn( `Run ${triggerResponse.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${triggerResponse.runId}`, + `Re-attach with: testsprite test wait ${triggerResponse.runId}, or cancel with: testsprite test cancel ${triggerResponse.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1): same partial- + // envelope shape as the timeout paths so stdout stays parseable, plus the + // honest "keeps running and billing" stderr line. Rethrow → index.ts + // renders the INTERRUPTED envelope and exits 128+signum. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${triggerResponse.runId} — interrupted (${err.signal})`); + const partial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, partial, opts.createContext, data => { + const p = data as typeof partial; + const lines = [`runId ${p.runId}`, `status ${p.status} (interrupted)`]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); + return lines.join('\n'); + }); + stderrFn(interruptDetachMessage(err, [triggerResponse.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -5441,16 +5631,21 @@ export async function runTestRun( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, ); + // BE detection: type hint (create-chain) OR beFallbackUsed (slow runs); on + // the standalone `test run ` path neither is set, so probe the test type + // once (text mode only, best-effort) — DEV-282. + const isBackend = await resolveRunCardIsBackend( + client, + opts.testId, + opts.output, + beFallbackUsed || opts.type === 'backend', + ); + printRunOrChain( out, withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), opts.createContext, - data => - renderRunResponseText(data as RunResponse, { - // BE detection: type hint (create-chain) OR beFallbackUsed (slow runs). - // This ensures fast BE runs terminal on first poll still render n/a. - isBackend: beFallbackUsed || opts.type === 'backend', - }), + data => renderRunResponseText(data as RunResponse, { isBackend }), ); // Surface the trigger requestId under --verbose/--debug or JSON mode so @@ -5460,10 +5655,10 @@ export async function runTestRun( stderrFn(`requestId: ${triggerRequestId}`); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { - // BE runs (resolved via the testId fallback) have no run-scoped artifact - // bundle — their failure bundle is addressed by testId, not runId. + // BE runs have no run-scoped artifact bundle — their failure bundle is + // addressed by testId, not runId. stderrFn( - beFallbackUsed + isBackend ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${finalRun.testId}' to download the bundle.` : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); @@ -5559,6 +5754,7 @@ export async function runTestWaitMany( const run = await pollRunUntilTerminal(client, runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -5570,6 +5766,10 @@ export async function runTestWaitMany( } catch (err) { if (err instanceof TimeoutError) return { kind: 'timeout' }; if (err instanceof RequestTimeoutError) throw err; + // Interrupt must reject the fan-out (handled at the collect point), not + // be flattened into a per-member 'error' outcome that would swallow the + // 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; if (err instanceof ApiError) return { kind: 'error', code: err.code, exitCode: err.exitCode }; return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; } @@ -5599,12 +5799,16 @@ export async function runTestWaitMany( if (opts.runIds.length === 0) resolve(); }); } catch (fanOutErr) { - if (fanOutErr instanceof RequestTimeoutError) { + if (fanOutErr instanceof RequestTimeoutError || fanOutErr instanceof InterruptError) { // Same contract as the batch pollers: leave stdout parseable before - // exiting 7. Members that already settled keep their real status; only + // exiting. Members that already settled keep their real status; only // the still-unfinished ids are marked running and named in the hint // (re-attaching to an already-terminal run would be a wasted command). - ticker.finalize('Multi-run wait — request timed out'); + ticker.finalize( + fanOutErr instanceof InterruptError + ? `Multi-run wait — interrupted (${fanOutErr.signal})` + : 'Multi-run wait — request timed out', + ); const partial = { results: opts.runIds.map((runId): CliMultiWaitResult => { const outcome = outcomes.get(runId); @@ -5620,7 +5824,13 @@ export async function runTestWaitMany( .filter(r => r.status === 'running' || r.status === 'timeout') .map(r => r.runId); if (unfinished.length > 0) { - stderrFn(`Re-attach with: testsprite test wait ${unfinished.join(' ')}`); + if (fanOutErr instanceof InterruptError) { + stderrFn(interruptDetachMessage(fanOutErr, unfinished)); + } else { + stderrFn( + `Re-attach with: testsprite test wait ${unfinished.join(' ')}, or cancel with: testsprite test cancel ${unfinished.join(' ')}`, + ); + } } } throw fanOutErr; @@ -5656,7 +5866,9 @@ export async function runTestWaitMany( .filter(r => r.status === 'timeout' || r.status.startsWith('error:')) .map(r => r.runId); if (unfinishedIds.length > 0) { - stderrFn(`Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}`); + stderrFn( + `Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}, or cancel with: testsprite test cancel ${unfinishedIds.join(' ')}`, + ); } // Worst-status exit: auth escalates (a rejected key fails every member the @@ -5684,6 +5896,190 @@ export async function runTestWaitMany( return payload; } +// --------------------------------------------------------------------------- +// DEV-331 piece 3 — `test cancel ` +// --------------------------------------------------------------------------- + +export interface RunTestCancelOptions extends CommonOptions { + runIds: string[]; +} + +/** One run's outcome in a multi-id `test cancel` summary. */ +export interface CliCancelResultRow { + runId: string; + status: 'cancelled' | 'alreadyCancelled' | 'conflict' | 'notFound' | 'error'; + /** Terminal status the run was already in, for a `conflict` row. */ + conflictStatus?: string; + error?: string; +} + +/** JSON payload for a multi-id `test cancel` — per piece-3 CXL-11. */ +export interface CliCancelSummary { + cancelled: string[]; + alreadyCancelled: string[]; + conflicts: Array<{ runId: string; status: string }>; + notFound: string[]; + /** + * Runs that errored for a reason other than 404/409 (e.g. auth, transport). + * Always present — an empty array on full success — so machine consumers + * can rely on a stable shape (DEV-331 codex finding 2). + */ + errors: Array<{ runId: string; message: string }>; +} + +/** + * `test cancel ` — DEV-331 piece 3. + * + * User-initiated cancel of one or more queued/running runs via + * `POST /api/cli/v1/runs/{runId}/cancel`. Naturally idempotent (D10): a + * repeat cancel of the same run is a 200 `alreadyCancelled` success, not an + * error. Dispatches serially (per-id volume is tiny — no batch endpoint, + * D8 "Batch-level cancel endpoint... YAGNI"). + * + * Single id: renders the returned run card (`status cancelled`); an + * `alreadyCancelled` response adds an `[advisory]` line. Exit code mirrors + * the server response directly — 0 on success (fresh or already-cancelled), + * 4 on 404 (unknown/cross-tenant), 6 on 409 (already terminal). + * + * Multi-id: prints a `{cancelled, alreadyCancelled, conflicts, notFound}` + * summary. Exit precedence (CXL-11): any `notFound` → 4 (outranks conflict — + * it signals a caller bug, wrong id/tenant); else any `conflicts` → 6; else 0. + */ +export async function runTestCancel( + opts: RunTestCancelOptions, + deps: TestDeps = {}, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + } + + if (opts.runIds.length === 1) { + const runId = opts.runIds[0]!; + const result = await client.cancelRun(runId); + out.print(result, data => { + const r = data as CancelRunResponse; + return renderRunResponseText(r); + }); + if (result.alreadyCancelled) { + stderrFn(`[advisory] run ${runId} was already cancelled`); + } + return result; + } + + const rows: CliCancelResultRow[] = []; + for (const runId of opts.runIds) { + try { + const result = await client.cancelRun(runId); + rows.push({ + runId, + status: result.alreadyCancelled ? 'alreadyCancelled' : 'cancelled', + }); + } catch (err) { + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + rows.push({ runId, status: 'notFound' }); + continue; + } + if (err instanceof ApiError && err.code === 'CONFLICT') { + const conflictStatus = + err.getDetail('status', (v): v is string => typeof v === 'string') ?? 'unknown'; + rows.push({ runId, status: 'conflict', conflictStatus }); + continue; + } + const message = err instanceof Error ? err.message : String(err); + rows.push({ runId, status: 'error', error: message }); + } + } + + const errorRows = rows.filter(r => r.status === 'error'); + const summary: CliCancelSummary = { + cancelled: rows.filter(r => r.status === 'cancelled').map(r => r.runId), + alreadyCancelled: rows.filter(r => r.status === 'alreadyCancelled').map(r => r.runId), + conflicts: rows + .filter(r => r.status === 'conflict') + .map(r => ({ runId: r.runId, status: r.conflictStatus ?? 'unknown' })), + notFound: rows.filter(r => r.status === 'notFound').map(r => r.runId), + errors: errorRows.map(r => ({ runId: r.runId, message: r.error ?? 'unknown error' })), + }; + + out.print(summary, data => renderCancelSummaryText(data as CliCancelSummary)); + + const parts = [ + `${summary.cancelled.length} cancelled`, + `${summary.alreadyCancelled.length} already cancelled`, + ]; + if (summary.conflicts.length > 0) parts.push(`${summary.conflicts.length} conflict`); + if (summary.notFound.length > 0) parts.push(`${summary.notFound.length} not found`); + if (errorRows.length > 0) parts.push(`${errorRows.length} error`); + stderrFn(`Cancel summary: ${parts.join(', ')}.`); + + // Exit precedence (CXL-11): notFound outranks conflict — a caller-side bug + // (wrong id / wrong tenant) is more actionable to surface than "it already + // finished". A bare transport/auth error on any member also fails loudly + // rather than being silently absorbed into a 0 exit. + if (summary.notFound.length > 0) { + throw new CLIError( + `${summary.notFound.length} run id${summary.notFound.length !== 1 ? 's' : ''} not found: ${summary.notFound.join(' ')}`, + 4, + ); + } + if (errorRows.length > 0) { + throw new CLIError( + `${errorRows.length} cancel request${errorRows.length !== 1 ? 's' : ''} failed: ${errorRows.map(r => r.runId).join(' ')}`, + 1, + ); + } + if (summary.conflicts.length > 0) { + throw new CLIError( + `${summary.conflicts.length} run${summary.conflicts.length !== 1 ? 's' : ''} already terminal: ${summary.conflicts.map(c => `${c.runId} (${c.status})`).join(', ')}`, + 6, + ); + } + return summary; +} + +function renderCancelSummaryText(summary: CliCancelSummary): string { + const lines: string[] = []; + for (const runId of summary.cancelled) lines.push(`${runId} cancelled`); + for (const runId of summary.alreadyCancelled) lines.push(`${runId} alreadyCancelled`); + for (const c of summary.conflicts) lines.push(`${c.runId} conflict (${c.status})`); + for (const runId of summary.notFound) lines.push(`${runId} notFound`); + for (const e of summary.errors ?? []) lines.push(`${e.runId} error (${e.message})`); + return lines.join('\n'); +} + +export function createTestCancelCommand(deps: TestDeps): Command { + const cancel = new Command('cancel'); + cancel + .argument('', 'one or more run ids to cancel') + .description( + 'Cancel one or more queued/running runs.\n' + + '\nCtrl-C during --wait only detaches — it does NOT cancel the server-side\n' + + 'run. This is the real stop button. No refund is issued for the credits\n' + + 'already charged at trigger time (D3); an in-flight Lambda finishes on its\n' + + 'own and its result is discarded once cancelled.\n' + + '\nExit codes:\n' + + ' 0 cancelled (fresh or already-cancelled — naturally idempotent)\n' + + ' 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict)\n' + + ' 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict\n' + + '\nMulti-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (runIds: string[], _cmdOpts: unknown, command: Command) => { + await runTestCancel( + { + ...resolveCommonOptions(command), + runIds, + }, + deps, + ); + }); + return cancel; +} + /** * `test wait ` — M3.3 piece-3. * @@ -5744,6 +6140,7 @@ export async function runTestWait( finalRun = await pollRunUntilTerminal(client, opts.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -5767,13 +6164,14 @@ export async function runTestWait( `runId ${p.runId}`, `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, ].join('\n'); }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${opts.runId}.`, - nextAction: `Resume polling: testsprite test wait ${opts.runId}`, + nextAction: `Resume polling: testsprite test wait ${opts.runId}, or cancel it: testsprite test cancel ${opts.runId}`, requestId: 'local', details: { runId: opts.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -5791,14 +6189,31 @@ export async function runTestWait( `runId ${p.runId}`, `status ${p.status} (request timed out)`, `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, ].join('\n'); }); stderrFn( `Run ${opts.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${opts.runId}`, + `Re-attach with: testsprite test wait ${opts.runId}, or cancel with: testsprite test cancel ${opts.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1) — see runTestRun. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${opts.runId} — interrupted (${err.signal})`); + const partial = { runId: opts.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (interrupted)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); + stderrFn(interruptDetachMessage(err, [opts.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -5809,15 +6224,24 @@ export async function runTestWait( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, ); + // `test wait` has no type hint; probe the test type once (text mode only, + // best-effort) so a backend run's card reads `n/a (backend)` — DEV-282. + const isBackend = await resolveRunCardIsBackend( + client, + finalRun.testId, + opts.output, + beFallbackUsed, + ); + out.print(withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), data => - renderRunResponseText(data as RunResponse, { isBackend: beFallbackUsed }), + renderRunResponseText(data as RunResponse, { isBackend }), ); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { - // BE runs (resolved via the testId fallback) have no run-scoped artifact - // bundle — their failure bundle is addressed by testId, not runId. + // BE runs have no run-scoped artifact bundle — their failure bundle is + // addressed by testId, not runId. stderrFn( - beFallbackUsed + isBackend ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${finalRun.testId}' to download the bundle.` : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); @@ -6276,6 +6700,7 @@ export async function runTestRunAll( const finalRun = await pollRunUntilTerminal(client, runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -6305,6 +6730,10 @@ export async function runTestRunAll( }, }; } + // Interrupt must reject the fan-out (the collect point below prints the + // partial for every dispatched run), never flatten into a per-member + // outcome that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; if (err instanceof RequestTimeoutError) { // Client-side per-request timeout during polling — classify as timeout // (exit 7) so the fan-out completes and stdout carries every runId. @@ -6340,24 +6769,45 @@ export async function runTestRunAll( let pollIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && pollIdx < pollable.length) { - const entry = pollable[pollIdx++]!; - inFlight++; - pollFreshAccepted(entry) - .then(result => { - freshRunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && pollIdx >= pollable.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && pollIdx < pollable.length) { + const entry = pollable[pollIdx++]!; + inFlight++; + pollFreshAccepted(entry) + .then(result => { + freshRunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && pollIdx >= pollable.length) resolve(); + }) + .catch(reject); + } } + startNext(); + if (pollable.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): stdout stays parseable — settled members + // keep their real status, unfinished ones are marked running — and the + // honest stderr line names every runId still executing (and billing). + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Batch run — interrupted (${fanOutErr.signal})`); + const settled = new Map(freshRunResults.map(r => [r.runId, r] as const)); + const partialResults = pollable.map( + (e): CliBatchRunFreshResult => + settled.get(e.runId) ?? { testId: e.testId, runId: e.runId, status: 'running' }, + ); + out.print( + { accepted: partialResults, conflicts, deferred, skippedFrontend, skippedIntegration }, + () => partialResults.map(r => `${r.runId} ${r.status}`).join('\n'), + ); + const unfinished = pollable.filter(e => !settled.has(e.runId)).map(e => e.runId); + if (unfinished.length > 0) stderrFn(interruptDetachMessage(fanOutErr, unfinished)); } - startNext(); - if (pollable.length === 0) resolve(); - }); + throw fanOutErr; + } ticker.finalize(); @@ -6411,7 +6861,10 @@ export async function runTestRunAll( error: { code: 'UNSUPPORTED', message: `${timedOut} run${timedOut !== 1 ? 's' : ''} timed out.`, - nextAction: timedOutRunIds.map(rid => `Resume: testsprite test wait ${rid}`).join('\n'), + nextAction: [ + ...timedOutRunIds.map(rid => `Resume: testsprite test wait ${rid}`), + ...timedOutRunIds.map(rid => `Cancel: testsprite test cancel ${rid}`), + ].join('\n'), requestId: 'local', details: { timedOutRunIds, timeoutSeconds: opts.timeoutSeconds }, }, @@ -6799,6 +7252,7 @@ export async function runTestRerun( return await pollRunUntilTerminal(client, member.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -6894,8 +7348,35 @@ export async function runTestRerun( const reattachHints = closureMembers .map(m => `testsprite test wait ${m.runId}`) .join('\n'); + const cancelHints = closureMembers + .map(m => `testsprite test cancel ${m.runId}`) + .join('\n'); stderrFn( - `Closure members are still in progress (request timed out). Re-attach with:\n${reattachHints}`, + `Closure members are still in progress (request timed out). Re-attach with:\n${reattachHints}\n` + + `Or cancel with:\n${cancelHints}`, + ); + throw fanOutErr; + } + // Graceful detach (DEV-331): same partial shape as the timeout path — + // SIG-6 requires the partial to list ALL dispatched runIds. + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Closure fan-out — interrupted (${fanOutErr.signal})`); + const dispatchedRunIds = closureMembers.map(m => ({ + runId: m.runId, + testId: m.testId, + role: m.role, + status: 'running' as const, + })); + out.print({ runId: namedRunId, status: 'running', closure: dispatchedRunIds }, () => + dispatchedRunIds + .map(m => `${m.role.padEnd(9)} ${m.testId} (runId: ${m.runId}) — running`) + .join('\n'), + ); + stderrFn( + interruptDetachMessage( + fanOutErr, + closureMembers.map(m => m.runId), + ), ); throw fanOutErr; } @@ -6937,7 +7418,7 @@ export async function runTestRerun( error: { code: 'UNSUPPORTED', message: `Timed out after ${opts.timeoutSeconds}s waiting for rerun ${namedRunId}.`, - nextAction: `Resume polling: testsprite test wait ${namedRunId}`, + nextAction: `Resume polling: testsprite test wait ${namedRunId}, or cancel it: testsprite test cancel ${namedRunId}`, requestId: 'local', details: { runId: namedRunId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -6957,7 +7438,10 @@ export async function runTestRerun( // as a whole was never observed to reach terminal. const timedOutMembers = closureFailures.filter(f => f.status === 'timeout'); if (timedOutMembers.length > 0) { - const resumeHints = timedOutMembers.map(f => `testsprite test wait ${f.runId}`).join('\n'); + const timedOutIds = timedOutMembers.map(f => f.runId); + const resumeHints = + timedOutIds.map(runId => `testsprite test wait ${runId}`).join('\n') + + `\nCancel instead: testsprite test cancel ${timedOutIds.join(' ')}`; throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', @@ -6995,6 +7479,7 @@ export async function runTestRerun( finalRun = await pollRunUntilTerminal(client, rerunResp.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -7012,7 +7497,7 @@ export async function runTestRerun( error: { code: 'UNSUPPORTED', message: `Timed out after ${opts.timeoutSeconds}s waiting for rerun ${rerunResp.runId}.`, - nextAction: `Resume polling: testsprite test wait ${rerunResp.runId}`, + nextAction: `Resume polling: testsprite test wait ${rerunResp.runId}, or cancel it: testsprite test cancel ${rerunResp.runId}`, requestId: 'local', details: { runId: rerunResp.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -7029,14 +7514,31 @@ export async function runTestRerun( `runId ${p.runId}`, `status ${p.status} (request timed out)`, `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, ].join('\n'); }); stderrFn( `Run ${rerunResp.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${rerunResp.runId}`, + `Re-attach with: testsprite test wait ${rerunResp.runId}, or cancel with: testsprite test cancel ${rerunResp.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1) — see runTestRun. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${rerunResp.runId} — interrupted (${err.signal})`); + const partial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (interrupted)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); + stderrFn(interruptDetachMessage(err, [rerunResp.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -7048,13 +7550,21 @@ export async function runTestRerun( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps replay)`, ); + // Probe the test type once (text mode only, best-effort) so a backend + // rerun's card reads `n/a (backend)` even when the fallback never fired + // (BE run rows finalize server-side now) — DEV-282. + const isBackend = await resolveRunCardIsBackend(client, testId, opts.output, beFallbackUsed); + out.print(withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), data => - renderRunResponseText(data as RunResponse, { isBackend: beFallbackUsed }), + renderRunResponseText(data as RunResponse, { isBackend }), ); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { + // BE reruns have no run-scoped artifact bundle — address by testId. stderrFn( - `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, + isBackend + ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${testId}' to download the bundle.` + : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); } @@ -7501,6 +8011,7 @@ export async function runTestRerun( const finalRun = await pollRunUntilTerminal(client, entry.runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -7530,6 +8041,10 @@ export async function runTestRerun( }, }; } + // Interrupt must reject the fan-out (the collect point below prints the + // partial for every dispatched run), never flatten into a per-member + // outcome that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; if (err instanceof RequestTimeoutError) { // Client-side per-request timeout during polling — classify as timeout // (exit 7) so the fan-out completes and stdout carries every runId. @@ -7565,24 +8080,44 @@ export async function runTestRerun( let acceptedIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && acceptedIdx < accepted.length) { - const entry = accepted[acceptedIdx++]!; - inFlight++; - pollAccepted(entry) - .then(result => { - rerunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && acceptedIdx >= accepted.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && acceptedIdx < accepted.length) { + const entry = accepted[acceptedIdx++]!; + inFlight++; + pollAccepted(entry) + .then(result => { + rerunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && acceptedIdx >= accepted.length) resolve(); + }) + .catch(reject); + } } + startNext(); + if (accepted.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): stdout stays parseable — settled members + // keep their real status, unfinished ones are marked running — and the + // honest stderr line names every runId still executing (and billing). + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Batch rerun — interrupted (${fanOutErr.signal})`); + const settled = new Map(rerunResults.map(r => [r.runId, r] as const)); + const partialResults = accepted.map( + (e): CliRerunResult => + settled.get(e.runId) ?? { testId: e.testId, runId: e.runId, status: 'running' }, + ); + out.print({ accepted: partialResults, deferred, conflicts, notFound }, () => + partialResults.map(r => `${r.runId} ${r.status}`).join('\n'), + ); + const unfinished = accepted.filter(e => !settled.has(e.runId)).map(e => e.runId); + if (unfinished.length > 0) stderrFn(interruptDetachMessage(fanOutErr, unfinished)); } - startNext(); - if (accepted.length === 0) resolve(); - }); + throw fanOutErr; + } ticker.finalize(); @@ -7646,6 +8181,7 @@ export async function runTestRerun( // `test wait` accepts exactly one run id — emit one command per // timed-out run so the hint is always valid. ...(timedOut > 0 ? stillRunning.map(rid => `Resume: testsprite test wait ${rid}`) : []), + ...(timedOut > 0 ? stillRunning.map(rid => `Cancel: testsprite test cancel ${rid}`) : []), ] .filter(Boolean) .join('\n'), @@ -8312,6 +8848,22 @@ export function createTestCommand(deps: TestDeps = {}): Command { .option('--name ', 'new human-readable test name') .option('--description ', 'new human description (≤ 2000 chars)') .option('--priority ', 'new priority — one of: p0, p1, p2, p3') + .option( + '--produces ', + 'BE only: variable name this test captures (repeatable). Drives dependency-aware wave ordering.', + (val: string, prev: string[]) => [...(prev ?? []), val], + [] as string[], + ) + .option( + '--needs ', + 'BE only: variable name this test consumes (repeatable). Declares an upstream producer dependency.', + (val: string, prev: string[]) => [...(prev ?? []), val], + [] as string[], + ) + .option( + '--category ', + "BE only: test category. Use 'teardown' or 'cleanup' to mark a final-wave cleanup test.", + ) .option( '--idempotency-key ', 'opaque idempotency token (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', @@ -8327,6 +8879,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { priority: parseEnumFlag(cmdOpts.priority, 'priority', CLI_CREATE_PRIORITIES) as | CliCreatePriority | undefined, + produces: cmdOpts.produces, + needs: cmdOpts.needs, + category: cmdOpts.category, idempotencyKey: cmdOpts.idempotencyKey, }, deps, @@ -8411,10 +8966,13 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' 4 test not found\n' + ' 5 validation error (e.g., bad --target-url, or positional + --all both set)\n' + ' 6 conflict (already running — see nextAction for the active runId)\n' + - ' 7 timeout — resume with: testsprite test wait \n' + + ' 7 timeout — resume with: testsprite test wait , ' + + 'or stop it with: testsprite test cancel \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + ' 11 rate limited — honor Retry-After\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C during --wait detaches only (the run keeps executing and billing);\n' + + 'stop it for real with: testsprite test cancel ', ) .option( '--target-url ', @@ -8574,7 +9132,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' is recorded as error: in its row and folded into exit 7)\n' + ' 7 timeout or per-member poll error — resume with: testsprite test wait \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C detaches only (the run keeps executing and billing); stop it for\n' + + 'real with: testsprite test cancel ', ) .option('--timeout ', `max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`) .option( @@ -8626,9 +9186,12 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' 4 test not found\n' + ' 5 validation error\n' + ' 6 conflict (already running — see nextAction for the active runId)\n' + - ' 7 timeout or deferred — resume with: testsprite test wait \n' + + ' 7 timeout or deferred — resume with: testsprite test wait , ' + + 'or stop it with: testsprite test cancel \n' + ' 11 rate limited — honor Retry-After\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C during --wait detaches only (the run keeps executing and billing);\n' + + 'stop it for real with: testsprite test cancel ', ) .option('--all', 'rerun all tests in the resolved project (requires --project)', false) .option( @@ -8802,6 +9365,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { test.addCommand(createTestPlanCommand(deps)); test.addCommand(createTestFailureCommand(deps)); test.addCommand(createTestArtifactCommand(deps)); + test.addCommand(createTestCancelCommand(deps)); return test; } @@ -8937,12 +9501,20 @@ export async function runFlaky( const finalRun = await pollRunUntilTerminal(client, runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, resolveAlternate, }); outcome = finalRun.status as FlakyOutcome; failureKind = finalRun.failureKind; } catch (err) { + // Graceful detach (DEV-331): clean up the ticker line, name the run + // still executing server-side, and let index.ts exit 128+signum. + if (err instanceof InterruptError) { + ticker.finalize(`Attempt ${i}/${opts.runs} — interrupted (${err.signal})`); + stderrFn(interruptDetachMessage(err, [runId])); + throw err; + } // A per-attempt deadline (poll TimeoutError) or a client-side request // timeout both count as a non-passing "timeout" outcome for this attempt. if (err instanceof TimeoutError || err instanceof RequestTimeoutError) { @@ -9015,6 +9587,9 @@ interface UpdateFlagOpts { name?: string; description?: string; priority?: string; + produces?: string[]; + needs?: string[]; + category?: string; idempotencyKey?: string; } @@ -9199,6 +9774,7 @@ function makeClient(opts: CommonOptions, deps: TestDeps): HttpClient { credentialsPath: deps.credentialsPath, fetchImpl: deps.fetchImpl, stderr: deps.stderr, + shutdownSignal: shutdownOf(deps).signal, }); } @@ -9506,6 +10082,16 @@ function renderTestText(t: CliTest): string { if (typeof t.planStepCount === 'number') { lines.push(`planSteps: ${t.planStepCount}`); } + // Surface backend dependency declarations when present. + if (Array.isArray(t.produces) && t.produces.length > 0) { + lines.push(`produces: ${t.produces.join(', ')}`); + } + if (Array.isArray(t.consumes) && t.consumes.length > 0) { + lines.push(`consumes: ${t.consumes.join(', ')}`); + } + if (t.category) { + lines.push(`category: ${t.category}`); + } lines.push(`createdAt: ${t.createdAt}`, `updatedAt: ${t.updatedAt}`); return lines.join('\n'); } @@ -9639,6 +10225,14 @@ function renderFailureContextText(ctx: CliFailureContext): string { lines.push(`evidence: ${ctx.failure.evidence.length} items (${breakdown})`); } if (ctx.result.videoUrl !== null) lines.push(`videoUrl: ${ctx.result.videoUrl}`); + // backend stdout + traceback (prefer the failure block, falling + // back to the embedded result; identical values). Bounded tail; full content + // in --output json / the written failure.json. + appendBackendArtifactLines( + lines, + ctx.failure.apiOutput ?? ctx.result.apiOutput, + ctx.failure.trace ?? ctx.result.trace, + ); return lines.join('\n'); } @@ -9760,6 +10354,9 @@ function renderResultText(r: CliLatestResult): string { lines.push(`summary: ${r.summary}`); if (r.videoUrl !== null) lines.push(`videoUrl: ${r.videoUrl}`); if (r.failureAnalysisUrl !== null) lines.push(`failureAnalysisUrl: ${r.failureAnalysisUrl}`); + // backend stdout + traceback (null/absent for FE, passed, and + // older backends). Full content always available via `--output json`. + appendBackendArtifactLines(lines, r.apiOutput, r.trace); if (r.analysis !== undefined) { // §6.5.1 (M2.1 piece 3) — render the inline analysis block under // the result summary. Only fires when the caller passed @@ -9783,6 +10380,44 @@ function renderResultText(r: CliLatestResult): string { * level null. Non-null wrappers render `kind=...` plus an optional * reference and indented rationale. */ +/** + * Render backend-test stdout + traceback in text mode. Shared by + * `renderResultText` and `renderFailureContextText`. Both are bounded to a + * tail (last {@link BACKEND_ARTIFACT_TAIL_LINES} lines) so a large stdout can't + * flood the terminal; the full, untruncated content is always in the + * `--output json` envelope. No-op when both are null/absent (FE / passed / + * older backends), so non-backend output stays byte-identical. + */ +const BACKEND_ARTIFACT_TAIL_LINES = 20; + +function appendArtifactTail( + lines: string[], + label: string, + value: string | null | undefined, +): void { + if (value == null || value === '') return; + const allLines = value.replace(/\n+$/, '').split('\n'); + const dropped = Math.max(0, allLines.length - BACKEND_ARTIFACT_TAIL_LINES); + const tail = dropped > 0 ? allLines.slice(-BACKEND_ARTIFACT_TAIL_LINES) : allLines; + const bytes = Buffer.byteLength(value, 'utf8'); + lines.push(''); + lines.push( + `${label} (${bytes} bytes${dropped > 0 ? `, showing last ${tail.length} lines` : ''}):`, + ); + for (const l of tail) lines.push(` ${l}`); + if (dropped > 0) + lines.push(` … ${dropped} earlier line(s) omitted — full content in --output json`); +} + +function appendBackendArtifactLines( + lines: string[], + apiOutput: string | null | undefined, + trace: string | null | undefined, +): void { + appendArtifactTail(lines, 'stdout', apiOutput); + appendArtifactTail(lines, 'trace', trace); +} + function appendFixTargetLines(lines: string[], fix: CliFixTarget | null, label: string): void { if (fix === null) { lines.push(`${label}— (analysis pipeline did not propose one)`); diff --git a/src/commands/test.wait.spec.ts b/src/commands/test.wait.spec.ts index c83195d..4830e50 100644 --- a/src/commands/test.wait.spec.ts +++ b/src/commands/test.wait.spec.ts @@ -10,7 +10,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import type { RunResponse } from '../lib/runs.types.js'; import { runTestWait } from './test.js'; @@ -1305,3 +1306,122 @@ describe('runTestWait — dashboardUrl on terminal output', () => { expect(printed.dashboardUrl).toBeUndefined(); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach on SIGINT/SIGTERM during test wait +// --------------------------------------------------------------------------- + +describe('runTestWait — InterruptError graceful detach (DEV-331)', () => { + function hangingFetch(): typeof globalThis.fetch { + return (async (_input: FetchInput, init: RequestInit = {}) => { + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + } + + it('SIG-2/SIG-4: emits partial JSON to stdout, honest billing line to stderr, rethrows exit 130', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const pending = runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: hangingFetch(), + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + shutdown, + }, + ); + // Simulate Ctrl-C while the long-poll fetch is in flight. + setTimeout(() => shutdown.interrupt('SIGINT'), 5); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + expect((err as InterruptError).signal).toBe('SIGINT'); + + // Stdout stays parseable and carries the runId for re-attach. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + + // The honest line: run keeps executing AND billing; re-attach hint. + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('testsprite test wait run_abc'); + }); + + it('SIG-3: SIGTERM maps to exit 143', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const pending = runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: hangingFetch(), + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGTERM'), 5); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(143); + }); + + it('arms the shutdown scope during the wait and disarms after a terminal result', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const fetchImpl = makeFetch(() => ({ body: makeRun('passed') })); + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + shutdown, + }, + ); + expect(shutdown.isArmed).toBe(false); // disarmed after the poll completes + }); +}); diff --git a/src/index.ts b/src/index.ts index bf9a9aa..ae3f944 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,7 @@ import { import { createProjectCommand } from './commands/project.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; -import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; +import { ApiError, CLIError, InterruptError, RequestTimeoutError } from './lib/errors.js'; import { installBrokenPipeGuard, installSignalHandlers } from './lib/interrupt.js'; import { Output, isOutputMode } from './lib/output.js'; import { maybeInstallProxyAgent } from './lib/proxy.js'; @@ -164,9 +164,12 @@ program.hook('preAction', (_thisCommand, actionCommand) => { } }); -// Clean process lifecycle: a clear message + conventional exit code on SIGINT / -// SIGTERM / SIGHUP (instead of Node's silent abrupt kill) so an interrupted -// `test run --wait` explains the run continues server-side; plus an EPIPE guard +// Clean process lifecycle (DEV-331 piece 1, errors.md §8.1): during a `--wait` +// poll the scope is armed — the first SIGINT/SIGTERM/SIGHUP aborts gracefully +// and the wait path prints an honest partial (the run KEEPS executing and +// billing server-side) + re-attach hint before exiting 128+signum; a second +// signal hard-exits. Outside an armed scope: a clear one-line message + +// immediate exit (instead of Node's silent abrupt kill). Plus an EPIPE guard // so piping to a reader that closes early (`| head`) exits cleanly instead of // dumping a raw `write EPIPE` stack. installSignalHandlers(); @@ -209,10 +212,44 @@ try { process.stderr.write(` granted: ${(granted as string[]).join(', ')}\n`); } } + // Surface the version gap on CLIENT_TOO_OLD so the user sees exactly what + // moved without parsing the message string. (No "latest" line — the npm + // update-notice is the single source of truth for the newest release.) + if (err.code === 'CLIENT_TOO_OLD') { + const your = err.getDetail('yourVersion'); + const min = err.getDetail('minVersion'); + if (typeof your === 'string' && typeof min === 'string') { + process.stderr.write(` your version: ${your}, minimum supported: ${min}\n`); + } + } } process.exit(err.exitCode); } const output = new Output(mode); + if (err instanceof InterruptError) { + // Graceful detach (DEV-331 piece 1, errors.md §8.1): the wait-path catch + // block already printed the honest partial + re-attach hint. Exit with + // the conventional 128+signum code; `INTERRUPTED` is deliberately outside + // the error catalog. Note: Ctrl-C does NOT cancel the server-side run. + if (mode === 'json') { + const envelope = { + error: { + code: 'INTERRUPTED', + message: err.message, + nextAction: + 'The server-side run (if any) keeps executing and billing. ' + + 'Re-attach with: testsprite test wait , or stop it with: testsprite test cancel ' + + '(runId is in the partial JSON on stdout).', + requestId: 'local', + details: { signal: err.signal }, + }, + }; + process.stderr.write(`${JSON.stringify(envelope, null, 2)}\n`); + } else { + process.stderr.write(`Error: ${err.message}\n`); + } + process.exit(err.exitCode); + } if (err instanceof RequestTimeoutError) { // Structured rendering for per-request timeouts: JSON mode emits a // machine-readable envelope; text mode emits the message with a hint. diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index ff20e8c..d954381 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -601,6 +601,25 @@ describe('resolveBundleDir', () => { const out = resolveBundleDir('/tmp/x/'); expect(out).toBe('/tmp/x'); }); + + // `C:\...` is only recognized as absolute by node:path when the process + // itself is running on win32 (path.isAbsolute/resolve are platform-native, + // not path.win32.* explicitly) — these two assertions are only meaningful + // under an actual Windows runtime, hence gated rather than run everywhere. + it.runIf(process.platform === 'win32')( + 'strips a trailing backslash (native Windows path)', + () => { + const out = resolveBundleDir('C:\\Users\\me\\bundle\\'); + expect(out).toBe('C:\\Users\\me\\bundle'); + }, + ); + + it.runIf(process.platform === 'win32')( + 'preserves a bare Windows drive root instead of truncating it to a drive-relative path', + () => { + expect(resolveBundleDir('C:\\')).toBe('C:\\'); + }, + ); }); describe('streamUrlToFile retry', () => { diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index a3c0808..f44549e 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -306,6 +306,23 @@ export function applyFailedOnly(ctx: CliFailureContext): CliFailureContext { }; } +/** + * Strip trailing path separators, tolerating both `/` and `\` since a + * native Windows `--out` value (typed or pasted from Explorer) commonly + * ends in a backslash. Preserves a bare drive root (`C:\`) — stripping + * its separator would turn it into `C:`, which Windows resolves as + * "current directory on drive C", not the drive root. + */ +function stripTrailingSeparators(rawPath: string): string { + if (rawPath.length <= 1) return rawPath; + let end = rawPath.length; + while (end > 1 && (rawPath[end - 1] === '/' || rawPath[end - 1] === '\\')) { + if (end === 3 && rawPath[1] === ':' && /[A-Za-z]/.test(rawPath[0]!)) break; + end--; + } + return rawPath.slice(0, end); +} + /** * Resolve the user-supplied `--out` path into an absolute directory. * Empty strings are rejected with `VALIDATION_ERROR` for consistency @@ -325,7 +342,7 @@ export function resolveBundleDir(rawPath: string): string { }, }); } - const trimmed = rawPath.endsWith('/') ? rawPath.slice(0, -1) : rawPath; + const trimmed = stripTrailingSeparators(rawPath); return isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed); } diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index 829410d..96d9dc0 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -24,8 +24,11 @@ import { REQUEST_TIMEOUT_MAX_MS, REQUEST_TIMEOUT_MIN_MS, } from './http.js'; +import { globalShutdown } from './interrupt.js'; import type { OutputMode } from './output.js'; import { createDryRunFetch } from './dry-run/fetch.js'; +import { noteServerVersion } from './version-notice.js'; +import { VERSION } from '../version.js'; export interface CommonOptions { profile: string; @@ -68,6 +71,12 @@ export interface ClientFactoryDeps { credentialsPath?: string; fetchImpl?: FetchImpl; stderr?: (line: string) => void; + /** + * Shutdown signal composed into every outgoing fetch (DEV-331 piece 1). + * Defaults to `globalShutdown.signal` so an armed SIGINT/SIGTERM aborts an + * in-flight request; tests inject their own controller's signal. + */ + shutdownSignal?: AbortSignal; } /** @@ -252,6 +261,7 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} onDebug: opts.debug ? (event: DebugEvent) => stderr(formatDryRunDebug(event)) : undefined, onTransition: opts.verbose ? (msg: string) => stderr(`[verbose] ${msg}`) : undefined, requestTimeoutMs, + shutdownSignal: deps.shutdownSignal ?? globalShutdown.signal, }); } @@ -274,7 +284,20 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} fetchImpl: deps.fetchImpl, onDebug: opts.debug ? (event: DebugEvent) => stderr(formatDebug(event)) : undefined, onTransition: opts.verbose ? (msg: string) => stderr(`[verbose] ${msg}`) : undefined, + // Warn once if the backend advertises a minimum supported version above + // this binary's. Gating (opt-out env, TTY, output mode, dry-run) lives in + // noteServerVersion; the client just forwards the observed headers. + onServerVersion: info => + noteServerVersion(info, { + currentVersion: VERSION, + env, + isTTY: process.stderr.isTTY === true, + outputMode: opts.output, + dryRun: opts.dryRun, + stderr, + }), requestTimeoutMs, + shutdownSignal: deps.shutdownSignal ?? globalShutdown.signal, }); } diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index a1838d6..078c653 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -333,6 +333,23 @@ describe('findSample', () => { updatedAt: expect.any(String), }); break; + case 'cancelRun': + // DEV-331 piece 3 — POST /runs/{runId}/cancel → CancelRunResponse + // (RunResponse shape + alreadyCancelled). + expect(body).toMatchObject({ + runId: expect.any(String), + testId: expect.any(String), + status: 'cancelled', + alreadyCancelled: false, + }); + break; + case 'deleteProject': + // DELETE /projects/{id} → CliDeleteProjectResponse shape. + expect(body).toMatchObject({ + projectId: expect.any(String), + deletedAt: expect.any(String), + }); + break; default: throw new Error(`Unexpected operationId in samples: ${e.operationId}`); } @@ -423,6 +440,22 @@ describe('findSample', () => { expect(body.stepSummary.failedCount).toBe(0); }); + // DEV-331 piece 3: POST /runs/{runId}/cancel must resolve to `cancelRun`, + // never fall through to the GET-only `getRun` entry despite sharing the + // `/runs/{runId}` path prefix — findSample filters by method first. + it('POST /runs/{runId}/cancel resolves cancelRun (not getRun)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz/cancel'); + expect(e?.operationId).toBe('cancelRun'); + const body = e?.body() as { status: string; alreadyCancelled: boolean; runId: string }; + expect(body.status).toBe('cancelled'); + expect(body.alreadyCancelled).toBe(false); + }); + + it('GET /runs/{runId}/cancel has no sample (cancel is POST-only)', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz/cancel'); + expect(e).toBeUndefined(); + }); + it('getTest sample carries priority field (G1a)', () => { const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_abc'); expect(e?.operationId).toBe('getTest'); diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 4c7cbb0..f6f5ec2 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -16,7 +16,11 @@ * If the CLI OpenAPI spec changes, both this file AND * `test/mock-backend/fixtures.ts` must be updated in the same PR. */ -import type { CliProject, CliUpdateProjectResponse } from '../../commands/project.js'; +import type { + CliProject, + CliUpdateProjectResponse, + CliDeleteProjectResponse, +} from '../../commands/project.js'; import type { CliBulkDeleteSummary, CliFailureContext, @@ -36,6 +40,7 @@ import type { BatchRerunResponse, BatchRunFreshResponse, ListRunsResponse, + CancelRunResponse, } from '../runs.types.js'; const SAMPLE_USER_ID = '11111111-1111-4111-8111-111111111111'; @@ -104,6 +109,7 @@ const me: MeResponse = { keyId: SAMPLE_KEY_ID, scopes: ['read:projects', 'read:tests', 'write:tests', 'run:tests'], env: 'development', + v3Enabled: true, }; const projects: CliProject[] = [ @@ -397,6 +403,11 @@ const ENTRIES: DryRunSampleEntry[] = [ updatedFields: ['name'], updatedAt: '2026-05-16T00:00:00.000Z', } satisfies CliUpdateProjectResponse), + // DELETE /projects/{id} (cascade delete project + tests + fixtures). + entry('deleteProject', 'DELETE', '/projects/{projectId}', { + projectId: SAMPLE_PROJECT_ID, + deletedAt: '2026-05-16T00:00:00.000Z', + } satisfies CliDeleteProjectResponse), entry('listTests', 'GET', '/tests', pageOf(tests)), entry('getTestCode', 'GET', '/tests/{testId}/code', testCode), entry('listTestSteps', 'GET', '/tests/{testId}/steps', pageOf(testSteps)), @@ -747,6 +758,36 @@ const ENTRIES: DryRunSampleEntry[] = [ }, ], } satisfies RunResponse), + // DEV-331 piece 3 — POST /runs/{runId}/cancel. Method-guarded in + // `findSample` (POST vs `getRun`'s GET), so this can't be shadowed by the + // broader `/runs/{runId}` pattern above despite sharing its path prefix. + // `alreadyCancelled: false` — a fresh cancel is the more instructive shape + // for a dry-run learner than the idempotent no-op. + entry('cancelRun', 'POST', '/runs/{runId}/cancel', { + runId: SAMPLE_RUN_ID, + testId: SAMPLE_TEST_ID_PASSED, + projectId: SAMPLE_PROJECT_ID, + userId: SAMPLE_USER_ID, + status: 'cancelled', + source: 'cli', + createdAt: '2026-05-15T19:32:00.000Z', + startedAt: '2026-05-15T19:32:05.000Z', + finishedAt: '2026-05-15T19:33:12.000Z', + codeVersion: 'v1', + targetUrl: SAMPLE_TARGET_URL, + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 8, + completed: 3, + passedCount: 3, + failedCount: 0, + }, + alreadyCancelled: false, + } satisfies CancelRunResponse), ]; function entry( diff --git a/src/lib/errors.test.ts b/src/lib/errors.test.ts index aa3926b..5c5ba66 100644 --- a/src/lib/errors.test.ts +++ b/src/lib/errors.test.ts @@ -59,6 +59,8 @@ describe('exitCodeFor', () => { ['UNAVAILABLE', 10], ['RATE_LIMITED', 11], ['INSUFFICIENT_CREDITS', 12], + ['FEATURE_GATED', 13], + ['CLIENT_TOO_OLD', 14], ['INTERNAL', 1], ] as const)('%s → exit %d', (code, expected) => { expect(exitCodeFor(code)).toBe(expected); @@ -138,6 +140,7 @@ describe('ApiError.fromEnvelope status fallback', () => { [403, 'AUTH_FORBIDDEN' as const], [404, 'NOT_FOUND' as const], [409, 'CONFLICT' as const], + [426, 'CLIENT_TOO_OLD' as const], [429, 'RATE_LIMITED' as const], [501, 'UNSUPPORTED' as const], [503, 'UNAVAILABLE' as const], @@ -157,6 +160,25 @@ describe('ApiError.fromEnvelope status fallback', () => { expect(err.exitCode).toBe(10); }); + it('recognizes a well-formed CLIENT_TOO_OLD (426) envelope and echoes the version details', () => { + const err = ApiError.fromEnvelope( + { + error: { + code: 'CLIENT_TOO_OLD', + message: 'Your CLI is older than the minimum supported version.', + nextAction: 'Upgrade with npm i -g @testsprite/testsprite-cli@latest.', + requestId: 'req_old', + details: { minVersion: '1.0.0', yourVersion: '0.9.0' }, + }, + }, + 426, + ); + // The code is in the union, so it is trusted directly — NOT remapped. + expect(err.code).toBe('CLIENT_TOO_OLD'); + expect(err.exitCode).toBe(14); + expect(err.details).toEqual({ minVersion: '1.0.0', yourVersion: '0.9.0' }); + }); + // Track A dogfood: NestJS raw 404 (route not registered) has the // shape `{ message, error: "Not Found", statusCode }`. Previously the // parser saw `obj.error = "Not Found"` (a string) and fell through to diff --git a/src/lib/errors.ts b/src/lib/errors.ts index f64d8e2..40b6c05 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -45,6 +45,12 @@ export const ERROR_CODES = [ // so the CLI can drop the client-side detection heuristic. 'FEATURE_GATED', 'UNSUPPORTED', + // The running CLI is older than the backend's minimum supported version. + // Backend emits this as HTTP 426 when version enforcement is enabled. Exit + // 14, non-retriable — a too-old client cannot self-heal by retrying; the + // user must upgrade. `nextAction` carries the upgrade instruction and + // `details` echoes { minVersion, latestVersion, yourVersion }. + 'CLIENT_TOO_OLD', 'INTERNAL', 'UNAVAILABLE', ] as const; @@ -77,6 +83,8 @@ export function exitCodeFor(code: ErrorCode): number { return 6; case 'UNSUPPORTED': return 7; + case 'CLIENT_TOO_OLD': + return 14; case 'UNAVAILABLE': return 10; case 'RATE_LIMITED': @@ -100,6 +108,41 @@ export class CLIError extends Error { } } +/** + * Termination signals with graceful handling, mapped to their conventional + * `128 + signum` exit code. sourceRef: POSIX signal numbers (SIGHUP=1, + * SIGINT=2, SIGTERM=15). Lives here (not interrupt.ts) so `InterruptError` + * needs no import cycle; `interrupt.ts` re-exports for back-compat. + */ +export const TERMINATION_EXIT_CODES = { + SIGINT: 130, // 128 + 2 + SIGTERM: 143, // 128 + 15 + SIGHUP: 129, // 128 + 1 +} as const; + +export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; + +/** + * User-initiated interrupt (SIGINT/SIGTERM/SIGHUP) observed while a + * graceful-detach scope (the `--wait` polling window) was armed — see + * `interrupt.ts::ShutdownController`. The `--wait` catch blocks render the + * honest partial envelope + re-attach hint, then rethrow to `index.ts`. + * + * Deliberately NOT in `ERROR_CODES` — on a signal the CLI + * exits 130/143 without consulting the error catalog. The JSON-mode stderr + * envelope uses the out-of-catalog code `"INTERRUPTED"`. Same client-local + * synthetic category as `RequestTimeoutError`. + */ +export class InterruptError extends CLIError { + readonly signal: TerminationSignal; + + constructor(signal: TerminationSignal) { + super(`Interrupted by ${signal}.`, TERMINATION_EXIT_CODES[signal]); + this.name = 'InterruptError'; + this.signal = signal; + } +} + export class NotImplementedError extends CLIError { constructor(commandPath: string) { super(`Command not yet implemented: ${commandPath}`, 2); @@ -314,6 +357,8 @@ function codeFromHttpStatus(status: number | undefined): ErrorCode { return 'PRECONDITION_FAILED'; case 413: return 'PAYLOAD_TOO_LARGE'; + case 426: + return 'CLIENT_TOO_OLD'; case 429: return 'RATE_LIMITED'; case 501: diff --git a/src/lib/failing-fe-resolver.spec.ts b/src/lib/failing-fe-resolver.spec.ts index d6524e7..1fe05a9 100644 --- a/src/lib/failing-fe-resolver.spec.ts +++ b/src/lib/failing-fe-resolver.spec.ts @@ -43,6 +43,40 @@ function makeItem( return { id, type, status, updatedAt }; } +/** + * URL-routing fetch stub for the preferredId direct-probe path: + * `GET /tests/{id}` (no query string) is answered from `preferred`; + * `GET /tests?...` list calls are answered from `pages` in order. + * Call counts are exposed so tests can assert which endpoints were hit. + */ +function makeRoutedFetch(opts: { + preferred?: { status: number; body: unknown }; + preferredThrows?: boolean; + pages: Array<{ items: TestListItem[]; nextToken: string | null }>; +}) { + let listCalls = 0; + let preferredCalls = 0; + const impl = (async (url: string | URL | Request) => { + const u = String(url); + if (/\/tests\/[^/?]+$/.test(u)) { + preferredCalls++; + if (opts.preferredThrows) throw new Error('ECONNRESET (direct probe)'); + const p = opts.preferred ?? { status: 404, body: { error: 'not found' } }; + return new Response(JSON.stringify(p.body), { + status: p.status, + headers: { 'Content-Type': 'application/json' }, + }); + } + const page = opts.pages[listCalls] ?? { items: [], nextToken: null }; + listCalls++; + return new Response(JSON.stringify(page), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof fetch; + return { impl, counts: () => ({ listCalls, preferredCalls }) }; +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -180,4 +214,163 @@ describe('resolveFailingFrontendTestId', () => { expect(result.testId).toBe('test_abc'); expect(result.reason).toContain('2026-05-27T09:00:00Z'); }); + + // DEV-393: dedicated pinned fixture must win over a fresher incidental + // failure — live-reproduced 2026-07-16 (a passingFrontendTestId fresh-run + // flip immediately outranked a dedicated "always red" fixture by + // updatedAt). See ResolverOptions.preferredId doc comment. + describe('preferredId (DEV-393 pinned-fixture preference)', () => { + test('prefers the pinned id over a fresher candidate when the direct probe is non-ok', async () => { + // Probe answers HTTP 500 (not ok, no throw) → falls through to the list + // scan, where the in-candidates preference must still pick the pinned id. + const routed = makeRoutedFetch({ + preferred: { status: 500, body: { error: 'internal' } }, + pages: [ + { + items: [ + makeItem('test_pinned', '2026-05-01T00:00:00Z'), + makeItem('test_incidental', '2026-05-20T00:00:00Z'), + ], + nextToken: null, + }, + ], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('Preferred pinned'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('falls back to freshest when the pinned id is not in the failed set', async () => { + // Probe returns 200 with a malformed body (no type/status fields) → + // falls through; pinned is absent from the list → freshest wins. + const routed = makeRoutedFetch({ + preferred: { status: 200, body: {} }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned_but_now_passing', + }); + expect(result.testId).toBe('test_incidental'); + }); + + test('behaves exactly as before when preferredId is not supplied', async () => { + const older = makeItem('test_old', '2026-05-01T00:00:00Z'); + const newer = makeItem('test_new', '2026-05-20T00:00:00Z'); + const fetchImpl = makeFetchReturning([{ items: [older, newer], nextToken: null }]); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: fetchImpl as typeof fetch, + }); + expect(result.testId).toBe('test_new'); + }); + + // Codex F3: the direct probe makes the preference immune to the maxPages + // list-scan cap — a pinned fixture beyond the last scanned page must + // still win, without any list call at all on the happy path. + test('direct probe wins without any list call when the pinned test is failed', async () => { + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { + ...makeItem('test_pinned', '2026-05-01T00:00:00Z'), + projectId: BASE_OPTS.projectId, + }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + maxPages: 1, + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('direct GET'); + expect(routed.counts()).toEqual({ listCalls: 0, preferredCalls: 1 }); + }); + + test('direct probe on a now-passing pinned test falls through to freshest', async () => { + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { + ...makeItem('test_pinned', '2026-05-25T00:00:00Z', 'frontend', 'passed'), + projectId: BASE_OPTS.projectId, + }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe error falls through to the list scan, where the pinned id still wins', async () => { + const routed = makeRoutedFetch({ + preferredThrows: true, + pages: [ + { + items: [ + makeItem('test_pinned', '2026-05-01T00:00:00Z'), + makeItem('test_incidental', '2026-05-20T00:00:00Z'), + ], + nextToken: null, + }, + ], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('Preferred pinned'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe on a pinned test from a DIFFERENT project falls through (codex round 2)', async () => { + // The pin points at a genuinely failed FE test — but in another + // project. The old list scan (projectId-filtered server-side) would + // never have returned it, so the direct probe must not let it win. + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { ...makeItem('test_pinned', '2026-05-01T00:00:00Z'), projectId: 'proj_OTHER' }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe 404 falls through and freshest wins when pinned is absent everywhere', async () => { + const routed = makeRoutedFetch({ + preferred: { status: 404, body: { error: 'not found' } }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned_deleted', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + }); }); diff --git a/src/lib/failing-fe-resolver.ts b/src/lib/failing-fe-resolver.ts index 14df49f..f8d6b5a 100644 --- a/src/lib/failing-fe-resolver.ts +++ b/src/lib/failing-fe-resolver.ts @@ -50,6 +50,21 @@ export interface ResolverOptions { * Each page uses pageSize=50, so 5 pages = 250 tests scanned. */ maxPages?: number; + /** + * DEV-393: the operator's pinned `failingFrontendTestId` (the static value + * from fixtures.local.json). "Freshest failed" alone is fragile in a shared + * project: any OTHER FE test that fails more recently than a dedicated, + * permanently-red fixture silently steals the slot (live-reproduced + * 2026-07-16 — a `passingFrontendTestId` fresh-run flip immediately + * outranked the dedicated fixture on `updatedAt`). When set, the pinned id + * is probed directly first (one GET /tests/{id} — immune to the maxPages + * list-scan cap); if it is currently `status:failed` it wins outright, + * regardless of timestamp. If the probe errors, the list scan below still + * prefers the pinned id when it appears among the candidates. Falls through + * to the existing freshest-wins behavior when unset or no longer failing — + * fully backward compatible. + */ + preferredId?: string; } export interface ResolverResult { @@ -67,9 +82,46 @@ export interface ResolverResult { * when no Failed test exists — never throws. */ export async function resolveFailingFrontendTestId(opts: ResolverOptions): Promise { - const { baseUrl, apiKey, projectId, maxPages = 5 } = opts; + const { baseUrl, apiKey, projectId, maxPages = 5, preferredId } = opts; const fetchImpl = opts.fetchImpl ?? fetch; + // DEV-393 (codex F3): probe the pinned fixture directly first, so the + // preference cannot be defeated by the maxPages list-scan cap (a pinned + // fixture beyond page `maxPages` would otherwise silently lose the slot to + // an incidental fresher failure). Any probe error or non-failed status + // falls through to the list scan; the in-candidates preference there + // remains as a second chance after a transient probe failure. + if (preferredId) { + try { + const resp = await fetchImpl(`${baseUrl}/tests/${encodeURIComponent(preferredId)}`, { + headers: { + 'x-api-key': apiKey, + 'x-request-id': `dev-e2e-resolver-preferred-${Date.now()}`, + accept: 'application/json', + }, + signal: AbortSignal.timeout(10_000), + }); + if (resp.ok) { + const test = (await resp.json()) as Partial & { projectId?: string }; + // The probe must honor the resolver's project scope (codex round 2): + // the list path filters by projectId server-side, so a pin that + // points at a failed FE test in a DIFFERENT project must not win + // here either. A response without a matching projectId (absent or + // mismatched) falls through to the project-scoped list scan. + if (test.type === 'frontend' && test.status === 'failed' && test.projectId === projectId) { + return { + testId: preferredId, + reason: + `Preferred pinned failingFrontendTestId (${preferredId}) confirmed status:failed ` + + `via direct GET (updatedAt=${test.updatedAt ?? 'unknown'}); list scan skipped`, + }; + } + } + } catch { + // Transient probe failure — fall through to the list scan below. + } + } + const candidates: TestListItem[] = []; let cursor: string | undefined; let pagesFetched = 0; @@ -134,6 +186,19 @@ export async function resolveFailingFrontendTestId(opts: ResolverOptions): Promi }; } + // DEV-393: a pinned, dedicated "always red" fixture wins over freshest + // whenever it is still in the failed set — see the doc comment on + // ResolverOptions.preferredId for why "freshest" alone is not enough. + if (preferredId) { + const pinned = candidates.find(c => c.id === preferredId); + if (pinned) { + return { + testId: pinned.id, + reason: `Preferred pinned failingFrontendTestId (${pinned.id}) is still status:failed (updatedAt=${pinned.updatedAt}); took priority over freshest-updatedAt candidate`, + }; + } + } + // Pick the most recently updated candidate — freshest failure is the // most likely to have a valid failure bundle attached. candidates.sort((a, b) => { diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index a9c1193..2baff2c 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { ApiError, RequestTimeoutError, TransportError } from './errors.js'; +import { ApiError, InterruptError, RequestTimeoutError, TransportError } from './errors.js'; import type { DebugEvent } from './http.js'; import { HttpClient, REQUEST_TIMEOUT_DEFAULT_MS, buildUrl, parseRetryAfter } from './http.js'; import { VERSION } from '../version.js'; @@ -29,7 +29,11 @@ function errorEnvelopeResponse(status: number, code: string, init: ResponseInit function makeClient( fetchImpl: typeof fetch, - options: { apiKey?: string | null; onDebug?: (e: DebugEvent) => void } = {}, + options: { + apiKey?: string | null; + onDebug?: (e: DebugEvent) => void; + onServerVersion?: (info: { minVersion?: string }) => void; + } = {}, ): HttpClient { const apiKey = 'apiKey' in options ? (options.apiKey ?? undefined) : 'sk-test'; return new HttpClient({ @@ -39,9 +43,73 @@ function makeClient( sleep: () => Promise.resolve(), random: () => 0, onDebug: options.onDebug, + onServerVersion: options.onServerVersion, }); } +describe('CLIENT_TOO_OLD (426)', () => { + it('is not retried — fails fast with the typed error', async () => { + const fetchImpl = vi.fn().mockResolvedValue(errorEnvelopeResponse(426, 'CLIENT_TOO_OLD')); + const client = makeClient(fetchImpl as unknown as typeof fetch); + + const err = await client.get('/me').catch((e: unknown) => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CLIENT_TOO_OLD'); + expect((err as ApiError).exitCode).toBe(14); + expect(fetchImpl).toHaveBeenCalledTimes(1); // no retry + }); +}); + +describe('onServerVersion hook', () => { + it('fires with the parsed floor header on a 2xx response', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue( + jsonResponse( + { ok: true }, + { + headers: { + 'content-type': 'application/json', + 'x-testsprite-cli-min-version': '1.0.0', + }, + }, + ), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/me'); + + expect(onServerVersion).toHaveBeenCalledWith({ minVersion: '1.0.0' }); + }); + + it('fires on a non-2xx response too (the floor header rides on every response)', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue( + errorEnvelopeResponse(404, 'NOT_FOUND', { + headers: { + 'content-type': 'application/json', + 'x-testsprite-cli-min-version': '1.0.0', + }, + }), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/tests/missing').catch(() => undefined); + + expect(onServerVersion).toHaveBeenCalledWith({ minVersion: '1.0.0' }); + }); + + it('does not fire when the floor header is absent', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue(jsonResponse({ ok: true })); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/me'); + + expect(onServerVersion).not.toHaveBeenCalled(); + }); +}); + describe('buildUrl', () => { it('handles trailing slashes and absolute paths', () => { expect(buildUrl('https://api.example.com/api/cli/v1', '/me')).toBe( @@ -641,3 +709,177 @@ describe('HttpClient per-request timeout', () => { expect(err).toBeInstanceOf(RequestTimeoutError); }); }); + +describe('HttpClient shutdown signal (DEV-331 graceful detach)', () => { + /** Stalled fetch that rejects with the effective signal's reason on abort. */ + function stalledFetch(callCounter?: { count: number }): typeof fetch { + return vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + if (callCounter) callCounter.count += 1; + return new Promise((_resolve, reject) => { + const signal = init?.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + if (reason instanceof Error) { + reject(reason); + return; + } + const err = new Error('aborted'); + err.name = 'AbortError'; + reject(err); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as unknown as typeof fetch; + } + + function makeShutdownClient( + shutdownSignal: AbortSignal, + callCounter?: { count: number }, + ): HttpClient { + return new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: stalledFetch(callCounter), + sleep: () => Promise.resolve(), + random: () => 0, + shutdownSignal, + }); + } + + it('aborts an in-flight fetch with the InterruptError reason (no retry, no re-wrap)', async () => { + const counter = { count: 0 }; + const shutdown = new AbortController(); + const client = makeShutdownClient(shutdown.signal, counter); + const pending = client.get('/runs/run_1'); + queueMicrotask(() => shutdown.abort(new InterruptError('SIGINT'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + expect((err as InterruptError).exitCode).toBe(130); + expect(counter.count).toBe(1); // never retried + }); + + it('classifies an anonymous AbortError as the interrupt when the shutdown signal fired', async () => { + // Some runtimes reject with a bare AbortError instead of the abort reason; + // rethrowIfAbort must still classify shutdown-first (never TransportError, + // never RequestTimeoutError). + const shutdown = new AbortController(); + const fetchImpl = vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => { + const err = new Error('aborted'); + err.name = 'AbortError'; + reject(err); + }, + { once: true }, + ); + }); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => Promise.resolve(), + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/runs/run_1'); + queueMicrotask(() => shutdown.abort(new InterruptError('SIGTERM'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + }); + + it('skips dispatch entirely when the shutdown signal is already aborted', async () => { + const counter = { count: 0 }; + const shutdown = new AbortController(); + shutdown.abort(new InterruptError('SIGINT')); + const client = makeShutdownClient(shutdown.signal, counter); + const err = await client.get('/me').catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(counter.count).toBe(0); + }); + + it('passes an InterruptError from a caller-composed signal through untouched (poll path, no shutdownSignal configured)', async () => { + // The polling loop composes the shutdown signal into its per-iteration + // caller signal; the fetch then rejects with the InterruptError reason + // even though the client itself has no shutdownSignal. + const counter = { count: 0 }; + const fetchImpl = stalledFetch(counter); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => Promise.resolve(), + random: () => 0, + }); + const caller = new AbortController(); + const pending = client.get('/runs/run_1', { signal: caller.signal }); + queueMicrotask(() => caller.abort(new InterruptError('SIGINT'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(counter.count).toBe(1); // no transport retry + }); +}); + +describe('HttpClient retry-delay sleep bails on shutdown (DEV-331 codex finding 1)', () => { + it('a shutdown during a transport-retry sleep rejects with InterruptError immediately', async () => { + // First fetch throws a transport error → the client schedules a retry + // sleep. The injected sleep NEVER resolves, so only the shutdown race can + // end it — without the bail, the interrupt would wait out the delay + // (e.g. a Retry-After: 60) before surfacing. + const shutdown = new AbortController(); + let calls = 0; + const fetchImpl = vi.fn(async () => { + calls += 1; + throw new Error('socket hang up'); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => new Promise(() => {}), // never resolves + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/me'); + // Let the first attempt fail and the retry sleep begin, then interrupt. + await new Promise(resolve => setTimeout(resolve, 10)); + shutdown.abort(new InterruptError('SIGINT')); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(calls).toBe(1); // the retry never dispatched + }); + + it('a shutdown during a RATE_LIMITED Retry-After sleep rejects with InterruptError immediately', async () => { + const shutdown = new AbortController(); + let calls = 0; + const fetchImpl = vi.fn(async () => { + calls += 1; + return errorEnvelopeResponse(429, 'RATE_LIMITED', { + headers: { 'content-type': 'application/json', 'retry-after': '60' }, + }); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => new Promise(() => {}), // never resolves + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/me'); + await new Promise(resolve => setTimeout(resolve, 10)); + shutdown.abort(new InterruptError('SIGTERM')); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + expect(calls).toBe(1); + }); +}); diff --git a/src/lib/http.ts b/src/lib/http.ts index 7a9051a..f0139ca 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; import type { ErrorCode } from './errors.js'; -import { ApiError, RequestTimeoutError, TransportError } from './errors.js'; +import { ApiError, InterruptError, RequestTimeoutError, TransportError } from './errors.js'; import { VERSION } from '../version.js'; import type { TriggerRunBody, @@ -14,6 +14,7 @@ import type { BatchRunFreshResponse, ListRunsQuery, ListRunsResponse, + CancelRunResponse, } from './runs.types.js'; export type FetchImpl = typeof globalThis.fetch; @@ -74,6 +75,14 @@ export interface HttpClientOptions { * Stays silent when absent; wired to stderr at `--verbose` level. */ onTransition?: (msg: string) => void; + /** + * Optional callback fired with the backend's supported-floor header + * (`X-TestSprite-CLI-Min-Version`) when present on a response. Fires on both + * success and error responses. The client forwards the value verbatim — it + * applies no policy itself; the caller (client-factory) decides whether to + * warn the user. + */ + onServerVersion?: (info: { minVersion?: string }) => void; /** * Per-request wall-clock timeout in milliseconds applied to every outgoing * fetch. The signal fires independently of any caller-supplied signal — the @@ -87,6 +96,16 @@ export interface HttpClientOptions { * request is well within the 120s default. */ requestTimeoutMs?: number; + /** + * Process-lifetime shutdown signal (DEV-331 piece 1). Composed into every + * outgoing fetch so an armed SIGINT/SIGTERM aborts an in-flight request + * (a `--wait` long-poll can sit inside a single fetch for minutes) instead + * of waiting out its window. Aborts with an `InterruptError` reason, which + * is classified before the timeout-vs-caller logic and is never retried or + * re-wrapped as `TransportError`. Defaults off; production wiring passes + * `globalShutdown.signal` via the client factory. + */ + shutdownSignal?: AbortSignal; } export interface RequestOptions { @@ -167,19 +186,42 @@ export class HttpClient { private readonly random: () => number; private readonly onDebug?: (event: DebugEvent) => void; private readonly onTransition?: (msg: string) => void; + private readonly onServerVersion?: (info: { minVersion?: string }) => void; private readonly requestTimeoutMs: number; + private readonly shutdownSignal?: AbortSignal; constructor(options: HttpClientOptions) { this.baseUrl = trimTrailingSlash(options.baseUrl); this.apiKey = options.apiKey; + this.shutdownSignal = options.shutdownSignal; this.fetchImpl = options.fetchImpl ?? globalThis.fetch.bind(globalThis); this.sleep = options.sleep ?? defaultSleep; this.random = options.random ?? Math.random; this.onDebug = options.onDebug; this.onTransition = options.onTransition; + this.onServerVersion = options.onServerVersion; this.requestTimeoutMs = options.requestTimeoutMs ?? REQUEST_TIMEOUT_DEFAULT_MS; } + /** + * Read the backend's supported-floor header off a response and forward it to + * `onServerVersion` when present. Never throws — a bad header must not break + * the request. Called on every response (success or error) so the advisory + * covers all paths. (The backend advertises only the floor; "latest" is + * resolved client-side via the npm update-notice.) + */ + private captureServerVersion(response: Response): void { + if (!this.onServerVersion) return; + try { + const minVersion = response.headers.get('x-testsprite-cli-min-version') ?? undefined; + if (minVersion !== undefined) { + this.onServerVersion({ minVersion }); + } + } catch { + // Header parsing is best-effort; never let it affect the request. + } + } + async get(path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta('GET', path, options).then(r => r.body); } @@ -382,6 +424,24 @@ export class HttpClient { }); } + /** + * POST /api/cli/v1/runs/{runId}/cancel + * User-initiated cancel of a queued/running run (DEV-331 piece 3). No + * body, no `Idempotency-Key` — the endpoint is naturally idempotent (D10): + * re-cancel → 200 `alreadyCancelled: true`; already-terminal (passed/ + * failed/blocked) → 409 CONFLICT; unknown/cross-tenant runId → 404. + * + * `retryOnConflict: false` — same rationale as `triggerRun`: a 409 here is + * a truthful terminal answer ("already finished"), not a transient + * snapshot conflict to paper over with a retry. + */ + async cancelRun(runId: string, options?: { signal?: AbortSignal }): Promise { + return this.post(`/runs/${encodeURIComponent(runId)}/cancel`, { + signal: options?.signal, + retryOnConflict: false, + }); + } + /** * Classify an error thrown while issuing OR reading a request. When it is an * abort/timeout and our per-request timeout signal fired (and the caller had @@ -397,6 +457,14 @@ export class HttpClient { requestId: string, effectiveSignal: AbortSignal = timeoutSignal, ): void { + // A user interrupt (DEV-331) outranks every other classification: once the + // shutdown signal fired, whatever error surfaced from the aborted fetch or + // body read is the interrupt. Throw its InterruptError reason so the wait + // paths can render the honest detach UX — never a RequestTimeoutError, and + // never fall through to the transport retry loop. + if (this.shutdownSignal?.aborted) { + throw this.shutdownSignal.reason; + } if (isAbortError(err) || isTimeoutError(err)) { const timeoutWon = timeoutSignal.aborted && @@ -422,6 +490,10 @@ export class HttpClient { let attempt = 0; while (true) { + // Bail before issuing (or re-issuing after a retry sleep) a request the + // user has already interrupted; the composed signal below would abort it + // immediately anyway, this just skips the wasted dispatch. + if (this.shutdownSignal?.aborted) throw this.shutdownSignal.reason; attempt += 1; this.debug({ kind: 'request', method, url, attempt, requestId }); const startedAt = Date.now(); @@ -436,8 +508,14 @@ export class HttpClient { // long-poll window (<=25s via ?waitSeconds), so it never bites polling. const requestTimeout = createRequestTimeout(this.requestTimeoutMs); const timeoutSignal = requestTimeout.signal; + const composedSignals = [timeoutSignal]; + if (options.signal != null) composedSignals.push(options.signal); + // Shutdown composition (DEV-331): an armed SIGINT/SIGTERM aborts the + // in-flight fetch immediately (reason: InterruptError) instead of + // letting a long-poll drain its window before the interrupt surfaces. + if (this.shutdownSignal != null) composedSignals.push(this.shutdownSignal); const effectiveSignal = - options.signal != null ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal; + composedSignals.length > 1 ? AbortSignal.any(composedSignals) : timeoutSignal; try { try { @@ -455,6 +533,13 @@ export class HttpClient { // A caller-supplied abort sets `name === 'AbortError'`. // We treat both abort variants together: if the timeout signal fired and // the caller hadn't already aborted, surface a clear RequestTimeoutError. + // A user interrupt is never retried and never re-wrapped as a + // TransportError (same passthrough discipline as RequestTimeoutError). + // The instanceof check matters even without `this.shutdownSignal`: + // the polling path composes the shutdown signal into its per- + // iteration caller signal, so the fetch can reject with the + // InterruptError reason directly (DEV-331). + if (err instanceof InterruptError) throw err; // A timeout/abort during the fetch itself: classify it (RequestTimeoutError // when our deadline fired; otherwise rethrow the caller's abort unmodified). this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); @@ -487,11 +572,15 @@ export class HttpClient { delayMs: decision.delayMs, }); requestTimeout.clear(); - await this.sleep(decision.delayMs); + await this.sleepBeforeRetry(decision.delayMs); continue; } const durationMs = Date.now() - startedAt; + // Surface the backend version-compatibility headers on every response + // (success or error) before branching, so the caller's advisory covers + // all paths. + this.captureServerVersion(response); if (response.ok) { this.debug({ kind: 'response', @@ -505,6 +594,8 @@ export class HttpClient { try { return { body: (await response.json()) as T, requestId, status: response.status }; } catch (err) { + // Interrupt passthrough (see the fetch catch above). + if (err instanceof InterruptError) throw err; // A timeout/abort can fire mid-body-read (headers received, stream stalls). this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); // Otherwise the successful response body was not valid JSON — a @@ -521,6 +612,8 @@ export class HttpClient { try { rawBody = await safeReadJson(response); } catch (err) { + // Interrupt passthrough (see the fetch catch above). + if (err instanceof InterruptError) throw err; // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse // errors), so a timeout fired mid-body-read on a non-OK response lands here. this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); @@ -559,7 +652,7 @@ export class HttpClient { delayMs: decision.delayMs, }); requestTimeout.clear(); - await this.sleep(decision.delayMs); + await this.sleepBeforeRetry(decision.delayMs); continue; } @@ -624,7 +717,7 @@ export class HttpClient { delayMs: decision.delayMs, }); requestTimeout.clear(); - await this.sleep(decision.delayMs); + await this.sleepBeforeRetry(decision.delayMs); } finally { requestTimeout.clear(); } @@ -652,6 +745,33 @@ export class HttpClient { return headers; } + /** + * Retry-delay sleep that bails the moment the shutdown signal fires + * (DEV-331, codex finding 1): a RATE_LIMITED `Retry-After: 60` or a + * transport backoff must not delay the honest-detach exit by up to a + * minute — reject with the InterruptError reason immediately. Mirrors + * poll.ts::sleepUnlessInterrupted. + */ + private sleepBeforeRetry(ms: number): Promise { + const signal = this.shutdownSignal; + if (signal == null) return this.sleep(ms); + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + this.sleep(ms).then( + () => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, + err => { + signal.removeEventListener('abort', onAbort); + reject(err instanceof Error ? err : new Error(String(err))); + }, + ); + }); + } + private debug(event: DebugEvent): void { if (this.onDebug) this.onDebug(event); } @@ -856,6 +976,9 @@ function apiRetryDecision( case 'UNSUPPORTED': case 'INSUFFICIENT_CREDITS': case 'FEATURE_GATED': + case 'CLIENT_TOO_OLD': + // CLIENT_TOO_OLD: retrying re-sends the same too-old client — it can only + // self-heal by upgrading, so fail fast with the upgrade guidance. return { retry: false, delayMs: 0 }; case 'CONFLICT': // Read paths (e.g. GET /failure) retry once: 409 = mid-mutation snapshot. diff --git a/src/lib/interrupt.test.ts b/src/lib/interrupt.test.ts index 1fcf8eb..bd20bba 100644 --- a/src/lib/interrupt.test.ts +++ b/src/lib/interrupt.test.ts @@ -1,8 +1,10 @@ import { EventEmitter } from 'node:events'; import { writeSync } from 'node:fs'; import { describe, expect, it, vi } from 'vitest'; +import { InterruptError } from './errors.js'; import { SIGINT_EXIT_CODE, + ShutdownController, TERMINATION_EXIT_CODES, formatInterruptMessage, installBrokenPipeGuard, @@ -30,32 +32,37 @@ describe('formatInterruptMessage', () => { }); }); +/** Fresh handler map + controller per case: the disarmed path exits on the + * FIRST signal, so sequential signals on one install take the second-signal + * hard-exit branch (by design — DEV-331 SIG-5). */ +function install(shutdown = new ShutdownController()) { + const handlers = new Map void>(); + const stderr: string[] = []; + const exit = vi.fn(); + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + stderr: line => stderr.push(line), + exit, + shutdown, + }); + return { handlers, stderr, exit, shutdown }; +} + describe('installSignalHandlers', () => { it('registers SIGINT, SIGTERM and SIGHUP with the conventional 128+signum exit codes', () => { - const handlers = new Map void>(); - const stderr: string[] = []; - const exit = vi.fn(); - - installSignalHandlers({ - on: (signal, handler) => handlers.set(signal, handler), - stderr: line => stderr.push(line), - exit, - }); - - expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); - - handlers.get('SIGINT')!(); - expect(exit).toHaveBeenLastCalledWith(130); - handlers.get('SIGTERM')!(); - expect(exit).toHaveBeenLastCalledWith(143); - handlers.get('SIGHUP')!(); - expect(exit).toHaveBeenLastCalledWith(129); - - // Each handler emits a leading blank line then the explanation. - expect(stderr[0]).toBe(''); - expect(stderr.join('\n')).toContain('Interrupted (SIGINT)'); - expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); - expect(stderr.join('\n')).toContain('Interrupted (SIGHUP)'); + for (const [signal, code] of [ + ['SIGINT', 130], + ['SIGTERM', 143], + ['SIGHUP', 129], + ] as const) { + const { handlers, stderr, exit } = install(); + expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); + handlers.get(signal)!(); + expect(exit).toHaveBeenLastCalledWith(code); + // Disarmed handler emits a leading blank line then the explanation. + expect(stderr[0]).toBe(''); + expect(stderr.join('\n')).toContain(`Interrupted (${signal})`); + } expect(SIGINT_EXIT_CODE).toBe(130); expect(TERMINATION_EXIT_CODES.SIGTERM).toBe(143); expect(TERMINATION_EXIT_CODES.SIGHUP).toBe(129); @@ -69,6 +76,7 @@ describe('installSignalHandlers', () => { installSignalHandlers({ on: (signal, handler) => handlers.set(signal, handler), exit, + shutdown: new ShutdownController(), }); handlers.get('SIGINT')!(); expect(exit).toHaveBeenCalledWith(130); @@ -78,6 +86,51 @@ describe('installSignalHandlers', () => { .join(''); expect(written).toContain('Interrupted (SIGINT)'); }); + + it('armed scope: first signal aborts with InterruptError and does NOT exit or print', () => { + const { handlers, stderr, exit, shutdown } = install(); + const disarm = shutdown.arm(); + handlers.get('SIGINT')!(); + + expect(exit).not.toHaveBeenCalled(); + expect(stderr).toEqual([]); + expect(shutdown.signal.aborted).toBe(true); + expect(shutdown.received).toBe('SIGINT'); + const reason = shutdown.signal.reason as InterruptError; + expect(reason).toBeInstanceOf(InterruptError); + expect(reason.signal).toBe('SIGINT'); + expect(reason.exitCode).toBe(130); + disarm(); + }); + + it('second signal during armed cleanup hard-exits with the second signal code (SIG-5)', () => { + const { handlers, exit, shutdown } = install(); + shutdown.arm(); + handlers.get('SIGINT')!(); + expect(exit).not.toHaveBeenCalled(); + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenCalledWith(143); + }); + + it('disposed scope reverts to the disarmed immediate-exit behavior', () => { + const { handlers, stderr, exit, shutdown } = install(); + const disarm = shutdown.arm(); + disarm(); + disarm(); // idempotent — double dispose must not underflow the counter + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenCalledWith(143); + expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); + }); + + it('nested arms (fan-out members) stay armed until the last disposer runs', () => { + const shutdown = new ShutdownController(); + const a = shutdown.arm(); + const b = shutdown.arm(); + a(); + expect(shutdown.isArmed).toBe(true); + b(); + expect(shutdown.isArmed).toBe(false); + }); }); describe('installBrokenPipeGuard', () => { diff --git a/src/lib/interrupt.ts b/src/lib/interrupt.ts index cc5b4d5..19edb09 100644 --- a/src/lib/interrupt.ts +++ b/src/lib/interrupt.ts @@ -19,22 +19,100 @@ * without spawning a subprocess or sending a real signal. */ +import { setMaxListeners } from 'node:events'; import { writeSync } from 'node:fs'; +import { InterruptError, TERMINATION_EXIT_CODES, type TerminationSignal } from './errors.js'; + +export { TERMINATION_EXIT_CODES, type TerminationSignal } from './errors.js'; + +/** Back-compat alias: SIGINT's conventional exit code. */ +export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; /** - * Termination signals handled, mapped to their conventional `128 + signum` - * exit code. sourceRef: POSIX signal numbers (SIGHUP=1, SIGINT=2, SIGTERM=15). + * Structural view of {@link ShutdownController} threaded through the DI + * surfaces (`TestDeps`, `PollOptions`) — commands and the polling loop need + * only these members, and tests can supply a lightweight fake. */ -export const TERMINATION_EXIT_CODES = { - SIGINT: 130, // 128 + 2 - SIGTERM: 143, // 128 + 15 - SIGHUP: 129, // 128 + 1 -} as const; +export interface ShutdownHandle { + /** Aborts (reason: `InterruptError`) when a termination signal arrives while armed. */ + readonly signal: AbortSignal; + /** Enter a graceful-detach scope. Returns the disposer that leaves it. */ + arm(): () => void; +} -export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; +/** + * Process-lifetime coordinator between the signal handler and the `--wait` + * polling paths (DEV-331 piece 1). + * + * Two modes, chosen by whether a graceful-detach scope is armed when the + * signal arrives: + * + * - **Armed** (inside `pollRunUntilTerminal`): the handler only aborts + * `signal` with an `InterruptError` — no I/O, no exit. The in-flight fetch + * and every backoff sleep bail immediately; the `--wait` catch blocks own + * the cleanup (finalize the ticker, print the honest partial envelope + + * re-attach hint, rethrow to `index.ts` → exit 130/143/129). + * - **Disarmed** (no wait in progress — prompts, one-shot commands, local + * FS work): the handler prints the generic explanation and exits + * immediately, preserving the pre-DEV-331 behavior. An abort nobody + * observes must never leave the process hanging at e.g. a readline prompt. + * + * A second signal while the armed cleanup is in flight is the documented + * escape hatch: immediate hard exit. + */ +export class ShutdownController { + private readonly controller = new AbortController(); + private armedCount = 0; + private receivedSignal: TerminationSignal | null = null; -/** Back-compat alias: SIGINT's conventional exit code. */ -export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; + constructor() { + // Every fetch and every poll iteration composes this signal via + // AbortSignal.any — a 50-run batch fan-out legitimately holds >10 + // concurrent listeners, so silence Node's MaxListeners warning. + setMaxListeners(0, this.controller.signal); + } + + get signal(): AbortSignal { + return this.controller.signal; + } + + /** The first termination signal received, or null if none yet. */ + get received(): TerminationSignal | null { + return this.receivedSignal; + } + + get isArmed(): boolean { + return this.armedCount > 0; + } + + /** + * Enter a graceful-detach scope (re-entrant: fan-out members overlap). + * Returns an idempotent disposer. + */ + arm(): () => void { + this.armedCount += 1; + let disposed = false; + return () => { + if (disposed) return; + disposed = true; + this.armedCount -= 1; + }; + } + + /** Record the signal and abort with an `InterruptError` carrying it. */ + interrupt(signal: TerminationSignal): void { + this.receivedSignal = signal; + this.controller.abort(new InterruptError(signal)); + } +} + +/** + * The process-wide instance: `index.ts` hands it to `installSignalHandlers`, + * and it is the default `shutdown` for `TestDeps` / `PollOptions` / + * `ClientFactoryDeps`, so production wiring is automatic. Tests inject their + * own `ShutdownController` (or a `ShutdownHandle` fake) instead. + */ +export const globalShutdown = new ShutdownController(); export function formatInterruptMessage(signal: TerminationSignal = 'SIGINT'): string { return ( @@ -50,11 +128,19 @@ export interface InterruptDeps { stderr?: (line: string) => void; /** Process exit. Defaults to `process.exit`. */ exit?: (code: number) => void; + /** Shutdown coordinator. Defaults to {@link globalShutdown}. */ + shutdown?: ShutdownController; } /** * Register handlers for SIGINT, SIGTERM and SIGHUP. Idempotent enough for a * single top-level call in `index.ts`; not designed to be installed twice. + * + * First signal, armed scope: abort-only — the `--wait` catch paths own the + * honest-detach UX and the exit (DEV-331 D1: Ctrl-C = detach, never cancel). + * First signal, disarmed: print the generic explanation + exit `128+signum`. + * Second signal (any mode): immediate hard exit — the escape hatch when the + * graceful cleanup itself wedges. */ export function installSignalHandlers(deps: InterruptDeps = {}): void { const on = @@ -75,9 +161,27 @@ export function installSignalHandlers(deps: InterruptDeps = {}): void { } }); const exit = deps.exit ?? ((code: number) => process.exit(code)); + const shutdown = deps.shutdown ?? globalShutdown; for (const signal of Object.keys(TERMINATION_EXIT_CODES) as TerminationSignal[]) { on(signal, () => { + if (shutdown.received !== null) { + // Second signal while graceful cleanup is in flight: hard exit now. + exit(TERMINATION_EXIT_CODES[signal]); + return; + } + if (shutdown.isArmed) { + // Graceful detach: abort only (sync, signal-safe — no I/O here so a + // pending stdout `drain` wait can settle); the armed catch paths + // finalize the ticker, print the partial + re-attach hint, and exit + // via index.ts with this signal's code. + shutdown.interrupt(signal); + return; + } + // Disarmed (no --wait in progress): legacy immediate exit. Record the + // signal first so a second one takes the hard-exit branch even when + // `exit` is injected and does not terminate (unit tests). + shutdown.interrupt(signal); // Blank line first so the message starts on its own row rather than // trailing the progress ticker's in-place line. stderr(''); diff --git a/src/lib/poll.spec.ts b/src/lib/poll.spec.ts index e6a14a1..93889eb 100644 --- a/src/lib/poll.spec.ts +++ b/src/lib/poll.spec.ts @@ -7,7 +7,8 @@ */ import { describe, expect, it } from 'vitest'; -import { ApiError } from './errors.js'; +import { ApiError, InterruptError } from './errors.js'; +import { ShutdownController } from './interrupt.js'; import { pollRunUntilTerminal, TimeoutError } from './poll.js'; import type { RunClient } from './poll.js'; import type { RunResponse } from './runs.types.js'; @@ -794,3 +795,104 @@ describe('pollRunUntilTerminal — resolveAlternate hook', () => { } }); }); + +// --------------------------------------------------------------------------- +// Graceful detach — shutdown handle (DEV-331 piece 1) +// --------------------------------------------------------------------------- + +describe('pollRunUntilTerminal — shutdown (SIGINT/SIGTERM graceful detach)', () => { + it('throws the InterruptError when the shutdown signal was already aborted (beats the deadline)', async () => { + const shutdown = new ShutdownController(); + shutdown.interrupt('SIGINT'); + // timeoutSeconds 0 → the deadline has also passed; the interrupt must win. + const err = await pollRunUntilTerminal(makeClient([makeRun('passed')]), RUN_ID, { + timeoutSeconds: 0, + sleep: instantSleep, + shutdown, + }).catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + }); + + it('aborts an in-flight long-poll fetch and surfaces InterruptError (not TimeoutError)', async () => { + // getRun hangs until the composed per-iteration signal aborts — the same + // contract as a real fetch. Flag-checking between iterations would never + // notice; only the signal composition can interrupt this. + const client: RunClient = { + getRun: (_runId, opts) => + new Promise((_resolve, reject) => { + opts?.signal?.addEventListener('abort', () => reject(opts.signal!.reason), { + once: true, + }); + }), + }; + const shutdown = new ShutdownController(); + const poll = pollRunUntilTerminal(client, RUN_ID, { + timeoutSeconds: 30, + sleep: instantSleep, + shutdown, + }); + queueMicrotask(() => shutdown.interrupt('SIGINT')); + const err = await poll.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + expect((err as InterruptError).exitCode).toBe(130); + }); + + it('bails out of a retryAfterSeconds sleep immediately on interrupt', async () => { + // The injected sleep never resolves — only the shutdown race can end it. + const neverSleep = () => new Promise(() => {}); + const client = makeClient([makeRun('running', { retryAfterSeconds: 60 }), makeRun('running')]); + const shutdown = new ShutdownController(); + const poll = pollRunUntilTerminal(client, RUN_ID, { + timeoutSeconds: 300, + sleep: neverSleep, + shutdown, + }); + await new Promise(resolve => setTimeout(resolve, 10)); // let the loop reach the sleep + shutdown.interrupt('SIGTERM'); + const err = await poll.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + expect((err as InterruptError).exitCode).toBe(143); + }); + + it('arms the graceful-detach scope for the poll duration and disarms after', async () => { + let armed = 0; + let disposedCount = 0; + const handle = { + signal: new AbortController().signal, + arm: () => { + armed += 1; + return () => { + disposedCount += 1; + }; + }, + }; + const run = await pollRunUntilTerminal(makeClient([makeRun('passed')]), RUN_ID, { + timeoutSeconds: 5, + sleep: instantSleep, + shutdown: handle, + }); + expect(run.status).toBe('passed'); + expect(armed).toBe(1); + expect(disposedCount).toBe(1); + }); + + it('disarms even when the poll throws (TimeoutError path)', async () => { + let disposedCount = 0; + const handle = { + signal: new AbortController().signal, + arm: () => () => { + disposedCount += 1; + }, + }; + const err = await pollRunUntilTerminal(makeClient([makeRun('running')]), RUN_ID, { + timeoutSeconds: 0, + sleep: instantSleep, + shutdown: handle, + }).catch(e => e); + expect(err).toBeInstanceOf(TimeoutError); + expect(disposedCount).toBe(1); + }); +}); diff --git a/src/lib/poll.ts b/src/lib/poll.ts index 0dfb0ac..92498e2 100644 --- a/src/lib/poll.ts +++ b/src/lib/poll.ts @@ -21,7 +21,8 @@ * Deadline exceeded → throw `TimeoutError`. */ -import { ApiError } from './errors.js'; +import { ApiError, InterruptError } from './errors.js'; +import type { ShutdownHandle } from './interrupt.js'; import type { RunResponse } from './runs.types.js'; import { isTerminalStatus } from './runs.types.js'; @@ -89,6 +90,16 @@ export interface PollOptions { elapsedMs: number, signal: AbortSignal, ) => Promise; + /** + * Graceful-detach coordinator (DEV-331 piece 1). While the poll runs, the + * scope is armed: a SIGINT/SIGTERM aborts `shutdown.signal` with an + * `InterruptError` instead of killing the process, and this loop surfaces + * it immediately — the in-flight long-poll fetch aborts (composed into the + * per-iteration signal) and every backoff/retry sleep bails early. The + * caller's catch block renders the honest partial + re-attach hint. + * Absent (tests, non-wait callers): behavior is unchanged. + */ + shutdown?: ShutdownHandle; } const LONG_POLL_WAIT_SECONDS = 25; @@ -108,9 +119,30 @@ export async function pollRunUntilTerminal( client: RunClient, runId: string, options: PollOptions, +): Promise { + // Arm the graceful-detach scope for the duration of the poll (DEV-331): + // while armed, a termination signal aborts instead of hard-killing the + // process, and the wait-path catch blocks own the honest detach UX. + const disarm = options.shutdown?.arm(); + try { + return await pollLoop(client, runId, options); + } finally { + disarm?.(); + } +} + +async function pollLoop( + client: RunClient, + runId: string, + options: PollOptions, ): Promise { const { timeoutSeconds, onTick, onTransition, resolveAlternate } = options; - const sleep = options.sleep ?? defaultSleep; + const shutdownSignal = options.shutdown?.signal; + const rawSleep = options.sleep ?? defaultSleep; + // Every sleep site (retryAfterSeconds, backoff schedule, not_yet_visible, + // 5xx retry) bails immediately on interrupt — a Ctrl-C must not sit out a + // 15s backoff before it is noticed. + const sleep = (ms: number): Promise => sleepUnlessInterrupted(rawSleep, ms, shutdownSignal); const startMs = Date.now(); const deadlineMs = startMs + timeoutSeconds * 1000; @@ -123,6 +155,9 @@ export async function pollRunUntilTerminal( let notYetVisibleRetries = 0; while (true) { + // Interrupt outranks the deadline: a Ctrl-C that raced the timeout must + // surface as the honest detach, not as a generic TimeoutError. + if (shutdownSignal?.aborted) throw shutdownSignal.reason; const now = Date.now(); if (now >= deadlineMs) { throw new TimeoutError(runId, timeoutSeconds); @@ -139,20 +174,33 @@ export async function pollRunUntilTerminal( const abortTimer = setTimeout(() => { abortController.abort(); }, remainingMs + TRANSPORT_CUSHION_MS); + // Compose the interrupt into the per-iteration signal: a `--wait` can sit + // inside one <=25s long-poll fetch (and the auto-raised per-request + // timeout means even longer for slow backends) — checking the flag + // between iterations is not enough, the in-flight fetch must abort. + const iterationSignal = + shutdownSignal != null + ? AbortSignal.any([abortController.signal, shutdownSignal]) + : abortController.signal; let run: RunResponse; try { if (useBackoff) { - run = await client.getRun(runId, { signal: abortController.signal }); + run = await client.getRun(runId, { signal: iterationSignal }); } else { const waitSeconds = Math.min(remainingSeconds, LONG_POLL_WAIT_SECONDS); - run = await client.getRun(runId, { waitSeconds, signal: abortController.signal }); + run = await client.getRun(runId, { waitSeconds, signal: iterationSignal }); } // Successful GET resets the consecutive-error counter. consecutiveErrors = 0; notYetVisibleRetries = 0; } catch (err) { clearTimeout(abortTimer); + // Interrupt classification precedes the timeout mapping: the composed + // signal makes the fetch reject on Ctrl-C, and that abort must surface + // as the InterruptError — not as a spurious TimeoutError. + if (err instanceof InterruptError) throw err; + if (shutdownSignal?.aborted) throw shutdownSignal.reason; // An AbortError from our per-iteration controller means the deadline // passed while the fetch was in flight — surface as TimeoutError. if (isAbortError(err)) { @@ -239,9 +287,16 @@ export async function pollRunUntilTerminal( } const altAbort = new AbortController(); const altTimer = setTimeout(() => altAbort.abort(), altRemainingMs); + // The alternate lookup aborts on interrupt too; its errors are swallowed + // by the fallback (best-effort), so the loop-top interrupt check above + // surfaces the InterruptError on the next iteration. + const altSignal = + shutdownSignal != null + ? AbortSignal.any([altAbort.signal, shutdownSignal]) + : altAbort.signal; let alternate: RunResponse | null = null; try { - alternate = await resolveAlternate(run, elapsedMs, altAbort.signal); + alternate = await resolveAlternate(run, elapsedMs, altSignal); } finally { clearTimeout(altTimer); } @@ -294,6 +349,36 @@ function defaultSleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } +/** + * Race a sleep against the shutdown signal: rejects with the signal's + * `InterruptError` reason the moment it fires, so no backoff/retry wait can + * delay the honest-detach UX. Wraps the injected `sleep` (tests keep their + * deterministic fakes); the underlying timer is left to fire harmlessly — + * the interrupt path exits the process long before it matters. + */ +function sleepUnlessInterrupted( + sleep: (ms: number) => Promise, + ms: number, + signal: AbortSignal | undefined, +): Promise { + if (signal == null) return sleep(ms); + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + sleep(ms).then( + () => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, + err => { + signal.removeEventListener('abort', onAbort); + reject(err instanceof Error ? err : new Error(String(err))); + }, + ); + }); +} + /** * Detects an AbortError thrown when an AbortSignal fires. * Works for native fetch AbortErrors as well as `AbortController.abort()` diff --git a/src/lib/runs.types.ts b/src/lib/runs.types.ts index 7e556fe..15501c9 100644 --- a/src/lib/runs.types.ts +++ b/src/lib/runs.types.ts @@ -225,6 +225,20 @@ export interface RunResponse { steps?: RunStepDto[] | null; } +// --------------------------------------------------------------------------- +// DEV-331 piece 3 — cancel wire types +// --------------------------------------------------------------------------- + +/** + * Response from `POST /api/cli/v1/runs/{runId}/cancel`. + * Same shape as `GET /runs/{runId}` (`status: "cancelled"`, verdict + * untouched) plus `alreadyCancelled` distinguishing a fresh cancel + * (naturally idempotent) from a no-op re-cancel. + */ +export interface CancelRunResponse extends RunResponse { + alreadyCancelled: boolean; +} + /** Terminal states from the RunStatus union. */ export const TERMINAL_RUN_STATUSES: ReadonlySet = new Set([ 'passed', diff --git a/src/lib/v3-advisory.test.ts b/src/lib/v3-advisory.test.ts new file mode 100644 index 0000000..124720f --- /dev/null +++ b/src/lib/v3-advisory.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; +import { routingLabel, V3_ROUTING_ADVISORY, emitV3RoutingAdvisory } from './v3-advisory.js'; + +describe('routingLabel', () => { + it('maps the boolean to v3 / v2', () => { + expect(routingLabel(true)).toBe('v3'); + expect(routingLabel(false)).toBe('v2'); + }); +}); + +describe('V3 routing advisory', () => { + it('names each open behavior gap (cancel, delete, target-url)', () => { + const text = V3_ROUTING_ADVISORY.join('\n'); + expect(text).toContain('test cancel'); + expect(text).toContain('test delete'); + expect(text).toContain('--target-url'); + }); + + it('emitV3RoutingAdvisory writes every line to the sink', () => { + const lines: string[] = []; + emitV3RoutingAdvisory(l => lines.push(l)); + expect(lines).toEqual(V3_ROUTING_ADVISORY); + }); +}); diff --git a/src/lib/v3-advisory.ts b/src/lib/v3-advisory.ts new file mode 100644 index 0000000..a8b75ed --- /dev/null +++ b/src/lib/v3-advisory.ts @@ -0,0 +1,25 @@ +/** + * Shared V3-routing text surfaces for `auth status` and `doctor`. + * + * `v3Enabled` on the `/me` response is the authoritative routing bit. When it + * is on, some commands behave differently while the V3 gaps stay open — the + * advisory names them. Copy lives here so both commands stay in sync. + */ + +/** One-word routing label for the text card. */ +export function routingLabel(v3Enabled: boolean): 'v3' | 'v2' { + return v3Enabled ? 'v3' : 'v2'; +} + +/** Consolidated advisory (stderr) emitted when V3 routing is on. */ +export const V3_ROUTING_ADVISORY: string[] = [ + '[advisory] V3 routing is on for this account. While these gaps are open:', + ' - `test cancel` may return 404', + ' - `test delete` may leave a zombie run', + ' - `--target-url` is ignored on frontend runs', +]; + +/** Write the advisory to a stderr sink, one line per call. */ +export function emitV3RoutingAdvisory(stderr: (line: string) => void): void { + for (const line of V3_ROUTING_ADVISORY) stderr(line); +} diff --git a/src/lib/version-notice.test.ts b/src/lib/version-notice.test.ts new file mode 100644 index 0000000..1ce473e --- /dev/null +++ b/src/lib/version-notice.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + formatBelowFloorNotice, + noteServerVersion, + resetBelowFloorNoticeState, + shouldWarnBelowFloor, + type VersionNoticeDeps, +} from './version-notice.js'; + +/** Baseline deps: all gates open, running version below the floor. */ +function baseDeps(overrides: Partial = {}): VersionNoticeDeps { + return { + currentVersion: '0.9.0', + env: {}, + isTTY: true, + outputMode: 'text', + dryRun: false, + ...overrides, + }; +} + +afterEach(() => { + resetBelowFloorNoticeState(); + vi.restoreAllMocks(); +}); + +describe('shouldWarnBelowFloor', () => { + it('warns when the running version is strictly below the floor', () => { + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps())).toBe(true); + }); + + it('does not warn when at or above the floor', () => { + expect(shouldWarnBelowFloor({ minVersion: '0.9.0' }, baseDeps())).toBe(false); // equal + expect(shouldWarnBelowFloor({ minVersion: '0.8.0' }, baseDeps())).toBe(false); // above + }); + + it('does not warn when no minVersion header was present', () => { + expect(shouldWarnBelowFloor({}, baseDeps())).toBe(false); + expect(shouldWarnBelowFloor({ minVersion: undefined }, baseDeps())).toBe(false); + }); + + it('does not warn on unparseable versions (garbage never warns)', () => { + expect(shouldWarnBelowFloor({ minVersion: 'not-a-version' }, baseDeps())).toBe(false); + expect( + shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ currentVersion: 'nope' })), + ).toBe(false); + }); + + it('is gated off by the opt-out env (any non-empty value)', () => { + expect( + shouldWarnBelowFloor( + { minVersion: '1.0.0' }, + baseDeps({ env: { TESTSPRITE_NO_UPDATE_NOTIFIER: '1' } }), + ), + ).toBe(false); + expect( + shouldWarnBelowFloor( + { minVersion: '1.0.0' }, + baseDeps({ env: { TESTSPRITE_NO_UPDATE_NOTIFIER: '0' } }), + ), + ).toBe(false); + }); + + it('is gated off under --output json, --dry-run, and non-TTY', () => { + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ outputMode: 'json' }))).toBe( + false, + ); + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ dryRun: true }))).toBe(false); + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ isTTY: false }))).toBe(false); + }); +}); + +describe('formatBelowFloorNotice', () => { + it('names the current version, the floor, and the npm upgrade command', () => { + const line = formatBelowFloorNotice('0.9.0', '1.0.0'); + expect(line).toContain('0.9.0'); + expect(line).toContain('minimum supported version 1.0.0'); + expect(line).toContain('npm install -g @testsprite/testsprite-cli'); + expect(line).toContain('TESTSPRITE_NO_UPDATE_NOTIFIER=1'); + }); + + it('does not name a target release (npm update-notice owns "latest")', () => { + const line = formatBelowFloorNotice('0.9.0', '1.0.0'); + expect(line).not.toContain('Upgrade to 1.'); + }); +}); + +describe('noteServerVersion', () => { + it('emits exactly one advisory line when below the floor', () => { + const stderr = vi.fn(); + noteServerVersion({ minVersion: '1.0.0' }, baseDeps({ stderr })); + expect(stderr).toHaveBeenCalledTimes(1); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('below the minimum supported')); + }); + + it('warns at most once per process', () => { + const stderr = vi.fn(); + const deps = baseDeps({ stderr }); + noteServerVersion({ minVersion: '1.0.0' }, deps); + noteServerVersion({ minVersion: '1.0.0' }, deps); + noteServerVersion({ minVersion: '1.0.0' }, deps); + expect(stderr).toHaveBeenCalledTimes(1); + }); + + it('stays silent when not below the floor', () => { + const stderr = vi.fn(); + noteServerVersion({ minVersion: '0.5.0' }, baseDeps({ stderr })); + expect(stderr).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/version-notice.ts b/src/lib/version-notice.ts new file mode 100644 index 0000000..0e492dc --- /dev/null +++ b/src/lib/version-notice.ts @@ -0,0 +1,111 @@ +/** + * Inline "your CLI is below the backend's minimum supported version" advisory. + * + * The backend advertises its supported floor on every `/api/cli/v1` response via + * the `X-TestSprite-CLI-Min-Version` header. Unlike the npm-registry update + * notice (`update-check.ts`, which runs in a Commander `preAction` hook before + * any HTTP request), this advisory reacts to a header observed *during* the + * request, so it is emitted from the HTTP layer's `onServerVersion` hook (wired + * in `client-factory.ts`) rather than the pre-request notifier — no cross-run + * cache needed. + * + * Distinct from the update notice on purpose, and non-redundant: this names the + * backend FLOOR and fires only when the running version is strictly below it (a + * serious "you will be rejected once enforcement is on" state); the update + * notice names the npm LATEST and fires whenever a newer release exists. Each + * owns its own number — the backend never advertises "latest". + * + * Reuses `compareSemver` and the `TESTSPRITE_NO_UPDATE_NOTIFIER` opt-out from + * `update-check.ts` so the gating and semver semantics stay consistent. + */ +import { compareSemver, UPDATE_CHECK_OPT_OUT_ENV } from './update-check.js'; +import { VERSION } from '../version.js'; + +/** Version-compatibility signal observed on a backend response (the floor). */ +export interface ServerVersionInfo { + minVersion?: string; +} + +export interface VersionNoticeDeps { + /** Version the running binary reports. Defaults to the built-in `VERSION`. */ + currentVersion?: string; + env?: NodeJS.ProcessEnv; + /** Whether stderr is an interactive terminal. */ + isTTY?: boolean; + /** The command's `--output` mode; `'json'` suppresses the advisory. */ + outputMode?: string; + /** Suppress under `--dry-run` (the dry-run fetch returns no real headers). */ + dryRun?: boolean; + /** Sink for the single advisory line. */ + stderr?: (line: string) => void; +} + +/** + * True when every gate passes and the running version is strictly below the + * advertised floor. Gates mirror the update notice: opt-out env, JSON output, + * dry-run, and non-TTY all suppress. Pure — no side effects, no process state. + */ +export function shouldWarnBelowFloor( + info: ServerVersionInfo, + deps: VersionNoticeDeps = {}, +): boolean { + const env = deps.env ?? process.env; + const currentVersion = deps.currentVersion ?? VERSION; + const isTTY = deps.isTTY ?? process.stderr.isTTY === true; + + const optOut = env[UPDATE_CHECK_OPT_OUT_ENV]; + if (optOut !== undefined && optOut !== '') return false; + if (deps.outputMode === 'json') return false; + if (deps.dryRun === true) return false; + if (!isTTY) return false; + + const minVersion = info.minVersion; + if (!minVersion) return false; + + // compareSemver returns -1 when the first arg is OLDER than the second. + // Unparseable input on either side compares as 0, so garbage never warns. + return compareSemver(currentVersion, minVersion) === -1; +} + +/** + * The single advisory line. Names the floor (the backend's authority) and the + * upgrade command — it deliberately does NOT name a target release; the npm + * update-notice is the single source of truth for the newest version. + */ +export function formatBelowFloorNotice(currentVersion: string, minVersion: string): string { + return ( + `Your testsprite-cli (${currentVersion}) is below the minimum supported version ${minVersion}. ` + + `Upgrade: npm install -g @testsprite/testsprite-cli ` + + `(disable this notice with ${UPDATE_CHECK_OPT_OUT_ENV}=1).` + ); +} + +/** Module-level guard: at most one below-floor advisory per process. */ +let warnedThisProcess = false; + +/** Test-only: reset the once-per-process guard between cases. */ +export function resetBelowFloorNoticeState(): void { + warnedThisProcess = false; +} + +/** + * Emit the below-floor advisory at most once per process. Wired to the HTTP + * client's `onServerVersion` hook. Never throws — an advisory must not break or + * delay the command it rides along with. + */ +export function noteServerVersion(info: ServerVersionInfo, deps: VersionNoticeDeps = {}): void { + try { + if (warnedThisProcess) return; + if (!shouldWarnBelowFloor(info, deps)) return; + + const minVersion = info.minVersion; + if (!minVersion) return; + + const currentVersion = deps.currentVersion ?? VERSION; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(formatBelowFloorNotice(currentVersion, minVersion)); + warnedThisProcess = true; + } catch { + // Advisory is best-effort; never surface its failures to the command. + } +} diff --git a/src/version.ts b/src/version.ts index f60fdfd..5b301a6 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.3.0'; +export const VERSION = '0.4.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 62efe5d..b0681c5 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -146,6 +146,14 @@ Commands: get Get a project by id create [options] Create a new project update [options] Update project metadata + delete [options] Permanently delete a project and everything under it (sub-projects, + their tests, and backend fixtures). Requires --confirm. + + Exit codes: + 0 success + 3 auth error + 4 project not found (or already deleted) + 5 validation error (e.g., missing --confirm) credential [options] Set the static backend credential injected into every backend test (Bearer token / API key / Basic token / @@ -260,11 +268,14 @@ Commands: 4 test not found 5 validation error (e.g., bad --target-url, or positional + --all both set) 6 conflict (already running — see nextAction for the active runId) - 7 timeout — resume with: testsprite test wait + 7 timeout — resume with: testsprite test wait , or stop it with: testsprite test cancel 10 transport/network failure (UNAVAILABLE) — retry the command 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get + + Ctrl-C during --wait detaches only (the run keeps executing and billing); + stop it for real with: testsprite test cancel wait [options] Wait for one or more runs to reach a terminal status. With several run-ids the runs are polled concurrently under one shared @@ -282,6 +293,9 @@ Commands: 10 transport/network failure (UNAVAILABLE) — retry the command On failure/blocked/cancelled, run: testsprite test artifact get + + Ctrl-C detaches only (the run keeps executing and billing); stop it for + real with: testsprite test cancel rerun [options] [test-ids...] Re-execute a test (or multiple) as a cheap replay — FE replays the saved script (no credit), BE re-runs the dependency closure. Exit codes: @@ -291,10 +305,13 @@ Commands: 4 test not found 5 validation error 6 conflict (already running — see nextAction for the active runId) - 7 timeout or deferred — resume with: testsprite test wait + 7 timeout or deferred — resume with: testsprite test wait , or stop it with: testsprite test cancel 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get + + Ctrl-C during --wait detaches only (the run keeps executing and billing); + stop it for real with: testsprite test cancel flaky [options] Repeatedly replay a test to measure stability and surface flakiness. Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. @@ -309,10 +326,51 @@ Commands: failure Export the latest-failure agent bundle artifact Download run-scoped artifact bundles (M3.3 piece-4) + cancel Cancel one or more queued/running runs. + + Ctrl-C during --wait only detaches — it does NOT cancel the server-side + run. This is the real stop button. No refund is issued for the credits + already charged at trigger time (D3); an in-flight Lambda finishes on its + own and its result is discarded once cancelled. + + Exit codes: + 0 cancelled (fresh or already-cancelled — naturally idempotent) + 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict) + 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict + + Multi-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}. help [command] display help for command " `; +exports[`--help snapshots > test cancel 1`] = ` +"Usage: testsprite test cancel [options] + +Cancel one or more queued/running runs. + +Ctrl-C during --wait only detaches — it does NOT cancel the server-side +run. This is the real stop button. No refund is issued for the credits +already charged at trigger time (D3); an in-flight Lambda finishes on its +own and its result is discarded once cancelled. + +Exit codes: + 0 cancelled (fresh or already-cancelled — naturally idempotent) + 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict) + 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict + +Multi-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}. + +Arguments: + run-id one or more run ids to cancel + +Options: + -h, --help display help for command + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test code get 1`] = ` "Usage: testsprite test code get [options] @@ -463,11 +521,14 @@ Exit codes: 4 test not found 5 validation error 6 conflict (already running — see nextAction for the active runId) - 7 timeout or deferred — resume with: testsprite test wait + 7 timeout or deferred — resume with: testsprite test wait , or stop it with: testsprite test cancel 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get +Ctrl-C during --wait detaches only (the run keeps executing and billing); +stop it for real with: testsprite test cancel + Options: --all rerun all tests in the resolved project (requires --project) (default: false) @@ -568,12 +629,15 @@ Exit codes: 4 test not found 5 validation error (e.g., bad --target-url, or positional + --all both set) 6 conflict (already running — see nextAction for the active runId) - 7 timeout — resume with: testsprite test wait + 7 timeout — resume with: testsprite test wait , or stop it with: testsprite test cancel 10 transport/network failure (UNAVAILABLE) — retry the command 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get +Ctrl-C during --wait detaches only (the run keeps executing and billing); +stop it for real with: testsprite test cancel + Options: --target-url override the project default env URL for this run (http/https only, no localhost/private IPs) diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 959f212..3b04712 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -7,7 +7,7 @@ * and runs `auth whoami` against the mock." */ -import { execFileSync, spawn } from 'node:child_process'; +import { spawn } from 'node:child_process'; import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { createServer } from 'node:http'; @@ -15,6 +15,7 @@ import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execNpm } from './helpers/execNpm.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, '..'); @@ -37,7 +38,7 @@ beforeAll(async () => { // existsSync skip we used to do here let `dist` rot under // refactors and gave false-green on `project list` once // already. - execFileSync('npm', ['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); server = createServer((req: IncomingMessage, res: ServerResponse) => { const url = req.url ?? '/'; if (url.startsWith('/api/cli/v1/projects/')) { diff --git a/test/e2e/signal.e2e.test.ts b/test/e2e/signal.e2e.test.ts new file mode 100644 index 0000000..6e03743 --- /dev/null +++ b/test/e2e/signal.e2e.test.ts @@ -0,0 +1,188 @@ +/** + * Local e2e tests for SIGINT/SIGTERM graceful detach during `--wait` + * (exit 130/143/129 per the documented signal contract). + * + * Spawns the real built binary (`dist/index.js`) against a local HTTP stub + * whose `GET /runs/{id}` long-poll hangs forever, sends a real signal to the + * child, and asserts the honest-detach contract: + * + * - stdout: parseable partial `{runId, status:"running"}` (JSON mode) + * - stderr: "keeps running (and billing)" + re-attach hint (+ INTERRUPTED + * envelope in JSON mode) + * - exit code 130 (SIGINT) / 143 (SIGTERM) + * + * Run via: `npm run test:e2e` (builds first). Excluded from `npm test`. + */ + +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { createServer, type Server } from 'node:http'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, '../..'); +const BIN_PATH = join(REPO_ROOT, 'dist', 'index.js'); + +const RUN_ID = 'run_sig_e2e_01'; + +let server: Server; +let baseUrl = ''; +/** Resolvers waiting for the next hanging /runs request to arrive. */ +const runRequestWaiters: Array<() => void> = []; + +beforeAll(async () => { + if (!existsSync(BIN_PATH)) { + throw new Error('dist/index.js not found — run `npm run test:e2e` which builds first.'); + } + server = createServer((req, res) => { + // Hang every request (long-poll / stalled-backend simulation): the CLI's + // abort must cut it. Signal any test waiting for the request to arrive. + runRequestWaiters.splice(0).forEach(fn => fn()); + req.on('close', () => res.destroy()); + }); + await new Promise(resolveListen => { + server.listen(0, '127.0.0.1', () => resolveListen()); + }); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('no server address'); + baseUrl = `http://127.0.0.1:${address.port}`; +}); + +afterAll(async () => { + await new Promise(resolveClose => { + server.close(() => resolveClose()); + server.closeAllConnections(); + }); +}); + +/** Resolves when the stub receives the next hanging GET /runs request. */ +function nextRunRequest(): Promise { + return new Promise(resolveWait => runRequestWaiters.push(resolveWait)); +} + +interface SpawnResult { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +} + +/** + * Spawn `testsprite test wait` against the hanging stub, deliver `signal` + * once the long-poll request is in flight, and collect the outcome. + */ +async function waitAndInterrupt( + signal: NodeJS.Signals, + extraArgs: string[] = [], +): Promise { + const child = spawn( + process.execPath, + [BIN_PATH, 'test', 'wait', RUN_ID, '--timeout', '120', ...extraArgs], + { + env: { + ...process.env, + TESTSPRITE_API_KEY: 'sk-e2e-signal', + TESTSPRITE_API_URL: baseUrl, + TESTSPRITE_NO_SKILL_WARNING: '1', + TESTSPRITE_NO_UPDATE_NOTIFIER: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString())); + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString())); + + const arrived = nextRunRequest(); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + resolveExit => { + child.on('exit', (code, exitSignal) => resolveExit({ code, signal: exitSignal })); + }, + ); + + await arrived; // the long-poll fetch is in flight — the armed window is open + await new Promise(r => setTimeout(r, 150)); // let the request settle into the poll loop + child.kill(signal); + + const { code, signal: exitSignal } = await exited; + return { code, signal: exitSignal, stdout, stderr }; +} + +describe('signal e2e — graceful detach during test wait (DEV-331)', () => { + it('SIG-1/SIG-2: SIGINT → exit 130, partial JSON on stdout, honest stderr hint', async () => { + const result = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(result.code).toBe(130); + + // stdout is a parseable partial naming the runId (file redirects never 0-byte). + const partial = JSON.parse(result.stdout) as { runId: string; status: string }; + expect(partial.runId).toBe(RUN_ID); + expect(partial.status).toBe('running'); + + // stderr: honest detach line + machine-readable INTERRUPTED envelope. + expect(result.stderr).toContain('Interrupted (SIGINT)'); + expect(result.stderr).toContain('billing'); + expect(result.stderr).toContain(`testsprite test wait ${RUN_ID}`); + expect(result.stderr).toContain('"code": "INTERRUPTED"'); + expect(result.stderr).toContain('"signal": "SIGINT"'); + }, 30_000); + + it('SIG-1 (text mode): SIGINT → exit 130, human-readable partial + hint', async () => { + const result = await waitAndInterrupt('SIGINT'); + expect(result.code).toBe(130); + expect(result.stdout).toContain(RUN_ID); + expect(result.stdout).toContain('running (interrupted)'); + expect(result.stderr).toContain('Interrupted (SIGINT)'); + expect(result.stderr).toContain('Error: Interrupted by SIGINT.'); + }, 30_000); + + it('SIG-3: SIGTERM → exit 143', async () => { + const result = await waitAndInterrupt('SIGTERM', ['--output', 'json']); + expect(result.code).toBe(143); + expect(result.stderr).toContain('Interrupted (SIGTERM)'); + expect(result.stderr).toContain('"signal": "SIGTERM"'); + }, 30_000); + + it('SIG-7: SIGINT during a non-wait command → immediate exit 130 with the generic explanation', async () => { + // `test list` is outside any armed --wait scope. The stub hangs its fetch; + // the disarmed handler must exit immediately with the generic explanation + // (no partial envelope — there is no runId to re-attach to). + const child = spawn(process.execPath, [BIN_PATH, 'test', 'list', '--project', 'p1'], { + env: { + ...process.env, + TESTSPRITE_API_KEY: 'sk-e2e-signal', + TESTSPRITE_API_URL: baseUrl, + TESTSPRITE_NO_SKILL_WARNING: '1', + TESTSPRITE_NO_UPDATE_NOTIFIER: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString())); + const exited = new Promise<{ code: number | null }>(resolveExit => { + child.on('exit', code => resolveExit({ code })); + }); + const arrived = nextRunRequest(); + await arrived; // the list fetch is in flight (disarmed — no poll running) + child.kill('SIGINT'); + const { code } = await exited; + expect(code).toBe(130); + expect(stderr).toContain('Interrupted (SIGINT)'); + expect(stderr).toContain('test wait'); + expect(stderr).not.toContain(' at '); // no stack trace / corrupted output + }, 30_000); + + it('SIG-8: detach then re-attach — the same runId can be waited on again (server unaffected)', async () => { + // First wait: interrupted. + const first = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(first.code).toBe(130); + // Re-attach: the stub receives a fresh long-poll for the SAME runId — + // proof the detach was client-side only. (We interrupt again to end it.) + const second = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(second.code).toBe(130); + const partial = JSON.parse(second.stdout) as { runId: string }; + expect(partial.runId).toBe(RUN_ID); + }, 60_000); +}); diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index ee89b3b..b6c23b4 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -13,6 +13,7 @@ import { execFileSync } from 'node:child_process'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { beforeAll, describe, expect, it } from 'vitest'; +import { execNpm } from './helpers/execNpm.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, '..'); @@ -43,11 +44,13 @@ const cases: Array<[string, string[]]> = [ // R5: regression guard for commands that gained new flag wording ['test create-batch', ['test', 'create-batch', '--help']], ['test run', ['test', 'run', '--help']], + // DEV-331 piece 3 + ['test cancel', ['test', 'cancel', '--help']], ]; describe('--help snapshots', () => { beforeAll(() => { - execFileSync('npm', ['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); }); for (const [name, args] of cases) { diff --git a/test/helpers/execNpm.ts b/test/helpers/execNpm.ts new file mode 100644 index 0000000..0535112 --- /dev/null +++ b/test/helpers/execNpm.ts @@ -0,0 +1,17 @@ +import { execFileSync } from 'node:child_process'; + +/** + * Cross-platform `npm` invocation for test `beforeAll` build steps. + * On Windows, `npm` is a `.cmd` shim rather than a directly executable + * binary — `execFileSync('npm', ...)` fails with `ENOENT` there unless + * `shell: true` lets the OS resolve the shim through PATHEXT. + */ +export function execNpm( + args: string[], + options: { cwd: string; stdio?: 'pipe' | 'inherit' | 'ignore' }, +): Buffer | string { + return execFileSync('npm', args, { + ...options, + shell: process.platform === 'win32', + }); +} From d2f0357ec1f9f08419768a40bd89beead8bc7cb3 Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Fri, 17 Jul 2026 16:44:17 -0700 Subject: [PATCH 064/117] =?UTF-8?q?fix(ci):=20migrate=20github-script=20to?= =?UTF-8?q?=20v8=20=E2=80=94=20repair=20the=20repo-wide=20broken=20gate=20?= =?UTF-8?q?check=20(#260)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub's runner infra force-upgrades actions/github-script@v7 to Node 24, where the script-side require('@actions/github') can no longer resolve. Every recent PR shows a red 'gate' check with 'Cannot find module @actions/github' regardless of whether it links an issue. - Pin actions/github-script to ed597411 (v8.0.0) in pr-triage, ci-nudge, issue-triage. - Replace require('@actions/github').getOctokit(APP_TOKEN) with new (github.constructor)({ auth: APP_TOKEN }) — v8.0.0 injects no getOctokit helper (verified against src/async-function.ts at the pinned SHA), and the wrapped require cannot resolve modules without a checkout. - Bump actions/stale to 1e223db (v10.4.0). Diagnosis first surfaced by @sandman-sh in #257 — thank you! This lands separately with a SHA pin and a require-free client construction. Co-authored-by: Zeshi Du Co-authored-by: Claude Fable 5 --- .github/workflows/ci-nudge.yml | 4 ++-- .github/workflows/issue-triage.yml | 2 +- .github/workflows/pr-triage.yml | 4 ++-- .github/workflows/stale.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml index f14461f..bd6f9ad 100644 --- a/.github/workflows/ci-nudge.yml +++ b/.github/workflows/ci-nudge.yml @@ -55,7 +55,7 @@ jobs: # back to github-actions[bot], defeating this bot's own purpose. permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: APP_TOKEN: ${{ steps.app-token.outputs.token }} with: @@ -65,7 +65,7 @@ jobs: const marker = process.env.MARKER; const appClient = process.env.APP_TOKEN - ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + ? new (github.constructor)({ auth: process.env.APP_TOKEN }) : null; async function write(fn) { if (appClient) { diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 0da78c8..53bd570 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -57,7 +57,7 @@ jobs: # of inheriting the App's full installation permissions. permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: # App token when available (→ testsprite-hob[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml index cfd114d..d505f0b 100644 --- a/.github/workflows/pr-triage.yml +++ b/.github/workflows/pr-triage.yml @@ -71,7 +71,7 @@ jobs: # fallback path (unused by this specific App-token mint). permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: APP_TOKEN: ${{ steps.app-token.outputs.token }} with: @@ -86,7 +86,7 @@ jobs: const author = pr.user.login; const appClient = process.env.APP_TOKEN - ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + ? new (github.constructor)({ auth: process.env.APP_TOKEN }) : null; // Prefer the App identity; fall back to github-actions[bot] when the // App is absent or lacks the Pull-requests permission (403/404). diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fbaa607..b670166 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -26,7 +26,7 @@ jobs: issues: write # comment + (un)label + close stale issues pull-requests: write # comment + (un)label + close stale PRs steps: - - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 + - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 with: # ── issues ────────────────────────────────────────────────────── days-before-issue-stale: 60 From 3abcf2d501533caa2e803b922825ab0d5239caa4 Mon Sep 17 00:00:00 2001 From: Joseph Jang Date: Fri, 17 Jul 2026 16:52:41 -0700 Subject: [PATCH 065/117] fix(doctor): validate output mode (#251) Co-authored-by: Yazan-O --- src/commands/doctor.test.ts | 43 ++++++++++++++++++++++++++++++++++++- src/commands/doctor.ts | 4 ++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index 7a508dd..7974947 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -9,8 +9,9 @@ import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { Command } from 'commander'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { CLIError } from '../lib/errors.js'; +import { ApiError, CLIError } from '../lib/errors.js'; import { writeProfile } from '../lib/credentials.js'; import type { DoctorDeps, DoctorReport } from './doctor.js'; import { createDoctorCommand, runDoctor } from './doctor.js'; @@ -56,6 +57,14 @@ function healthyDeps(credentialsPath: string, extra: Partial = {}): }; } +function makeDoctorProgram(deps: DoctorDeps = {}): Command { + const program = new Command(); + program.exitOverride(); + program.option('--output ', 'output', 'text'); + program.addCommand(createDoctorCommand(deps)); + return program; +} + let credentialsPath: string; beforeEach(() => { @@ -271,4 +280,36 @@ describe('createDoctorCommand wiring', () => { it('--help describes the diagnostic', () => { expect(createDoctorCommand().helpInformation()).toContain('Diagnose'); }); + + it('rejects invalid --output with the shared VALIDATION_ERROR', async () => { + const rejection = await makeDoctorProgram() + .parseAsync(['node', 'ts', '--output', 'yaml', 'doctor']) + .catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(ApiError); + expect(rejection).toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + nextAction: 'Flag `--output` is invalid: must be one of: json, text.', + }); + }); + + it('accepts valid --output modes through command wiring', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + for (const mode of ['text', 'json'] as const) { + const { capture, deps } = makeCapture(); + await makeDoctorProgram({ ...healthyDeps(credentialsPath), ...deps }).parseAsync([ + 'node', + 'ts', + '--output', + mode, + 'doctor', + ]); + const raw = capture.stdout.join(''); + if (mode === 'json') { + expect((JSON.parse(raw) as DoctorReport).failures).toBe(0); + } else { + expect(raw).toContain('All checks passed.'); + } + } + }); }); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index cceeb13..40c1ac8 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -24,7 +24,7 @@ import { import { loadConfig } from '../lib/config.js'; import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; import { VERSION } from '../version.js'; @@ -287,7 +287,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, From 3c3a2e05b976abc7f4e5f823a55a78c9cf16ad25 Mon Sep 17 00:00:00 2001 From: Joseph Jang Date: Fri, 17 Jul 2026 16:53:25 -0700 Subject: [PATCH 066/117] Recovered: fix(bundle): atomic re-commit via per-entry aside (#196 by @SahilRakhaiya05) (#246) * fix(bundle): atomic re-commit via per-entry aside with rollback * fix(bundle): move meta.json aside before other bundle entries during commit --------- Co-authored-by: SahilRakhaiya05 --- src/lib/bundle.commit.test.ts | 124 ++++++++++++++++++++++++++ src/lib/bundle.ts | 160 +++++++++++++++++++--------------- 2 files changed, 212 insertions(+), 72 deletions(-) create mode 100644 src/lib/bundle.commit.test.ts diff --git a/src/lib/bundle.commit.test.ts b/src/lib/bundle.commit.test.ts new file mode 100644 index 0000000..83cbce8 --- /dev/null +++ b/src/lib/bundle.commit.test.ts @@ -0,0 +1,124 @@ +import type * as NodeFsPromises from 'node:fs/promises'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const renameMock = vi.hoisted(() => vi.fn()); +const rmMock = vi.hoisted(() => vi.fn()); + +vi.mock('node:fs/promises', async importOriginal => { + const actual = (await importOriginal()) as typeof NodeFsPromises; + return { + ...actual, + rename: renameMock, + rm: rmMock, + }; +}); + +const { commitBundle } = await import('./bundle.js'); + +describe('commitBundle', () => { + let realRename: typeof NodeFsPromises.rename; + let realRm: typeof NodeFsPromises.rm; + + beforeEach(async () => { + const actual = (await vi.importActual('node:fs/promises')) as typeof NodeFsPromises; + realRename = actual.rename; + realRm = actual.rm; + renameMock.mockImplementation(realRename); + rmMock.mockImplementation(realRm); + }); + + afterEach(() => { + renameMock.mockReset(); + rmMock.mockReset(); + }); + + async function withTempParent(run: (parent: string) => Promise): Promise { + const parent = mkdtempSync(join(tmpdir(), 'bundle-commit-parent-')); + try { + await run(parent); + } finally { + await realRm(parent, { recursive: true, force: true }).catch(() => undefined); + } + } + + function seedBundleDirs(parent: string): { dir: string; tmpDir: string; files: string[] } { + const dir = join(parent, 'bundle'); + const tmpDir = join(dir, '.tmp'); + + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'notes.txt'), 'foreign notes\n', 'utf8'); + mkdirSync(join(dir, 'steps'), { recursive: true }); + writeFileSync(join(dir, 'meta.json'), '{"snapshotId":"snap_old"}\n', 'utf8'); + writeFileSync(join(dir, 'steps', '01-evidence.json'), '{"step":1}\n', 'utf8'); + + mkdirSync(join(tmpDir, 'steps'), { recursive: true }); + writeFileSync(join(tmpDir, 'meta.json'), '{"snapshotId":"snap_new"}\n', 'utf8'); + writeFileSync(join(tmpDir, 'result.json'), '{}\n', 'utf8'); + writeFileSync(join(tmpDir, 'steps', '01-evidence.json'), '{"step":9}\n', 'utf8'); + + return { dir, tmpDir, files: ['result.json', 'meta.json', 'steps/01-evidence.json'] }; + } + + it('rolls back to the prior complete bundle when a staged rename fails', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + renameMock.mockImplementation(async (oldPath, newPath) => { + const dest = String(newPath); + if (dest.endsWith('result.json') && !dest.includes('.aside.')) { + throw Object.assign(new Error('simulated install failure'), { code: 'EACCES' }); + } + return realRename(oldPath, newPath); + }); + + await expect(commitBundle(tmpDir, dir, files)).rejects.toThrow('simulated install failure'); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_old"}\n'); + expect(readFileSync(join(dir, 'steps', '01-evidence.json'), 'utf8')).toBe('{"step":1}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + const leftovers = readdirSync(parent).filter(name => name.includes('.aside.')); + expect(leftovers).toEqual([]); + }); + }); + + it('preserves foreign files while installing the new bundle on success', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + await expect(commitBundle(tmpDir, dir, files)).resolves.toBeUndefined(); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_new"}\n'); + expect(readFileSync(join(dir, 'steps', '01-evidence.json'), 'utf8')).toBe('{"step":9}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + expect(existsSync(join(dir, 'result.json'))).toBe(true); + }); + }); + + it('keeps the new bundle when post-commit aside cleanup fails', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + rmMock.mockImplementation(async (path, options) => { + if (String(path).includes('.aside.')) { + throw Object.assign(new Error('simulated aside cleanup failure'), { code: 'EACCES' }); + } + return realRm(path, options); + }); + + await expect(commitBundle(tmpDir, dir, files)).resolves.toBeUndefined(); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_new"}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + }); + }); +}); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index f44549e..0685fd7 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -33,12 +33,13 @@ * when the agent re-runs. M3 may add resume. */ +import { randomUUID } from 'node:crypto'; import { mkdir, mkdtemp, readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'; import type { Writable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import type { ReadableStream as NodeReadableStream } from 'node:stream/web'; import { createWriteStream } from 'node:fs'; -import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'; import type { CliFailureContext, CliTestStep } from '../commands/test.js'; import { ApiError, TransportError, localValidationError } from './errors.js'; import { requireEnum } from './validate.js'; @@ -536,30 +537,6 @@ async function freshTmpDir(dir: string): Promise { return tmpDir; } -/** - * Rename `/` → `/` for every file in `files`. - * - * Critical ordering for atomicity (the §3 "agent-safe" contract): - * - * 1. **Remove the OLD `meta.json` first.** The bundle's completion - * signal is `meta.json`'s presence; an agent reading `` while - * we're mutating it must see "no meta → bundle absent or - * mid-write" rather than "meta points at a snapshot that's already - * been partially overwritten." Removing the old meta is what - * makes the rest of the swap safe to do in place. - * 2. Wipe stale top-level files (e.g. an old `video.mp4` when the new - * bundle has no video). Without this, a fresh bundle could ship - * with a stale video lingering at the top level. - * 3. Replace `/steps/` wholesale. - * 4. Rename top-level files into place. - * 5. **Rename `meta.json` LAST.** Its visible presence is the atomic - * completion signal; until step 5 lands, agents see "incomplete." - * - * The window between (1) and (5) is bounded by a handful of `rename` - * syscalls — small enough that a SIGKILL there is rare, and any agent - * caught reading the dir during it sees no meta and refuses to consume - * (per §7.3). That's what we want. - */ /** * Whether a top-level directory entry belongs to the bundle format — * i.e. something a prior `writeBundle` could have produced and this @@ -584,62 +561,101 @@ export function isBundleOwnedEntry(entry: string): boolean { return /^code\.[A-Za-z0-9]+$/.test(entry); } -async function commitBundle( +/** + * Atomically install the complete bundle staged in `tmpDir` into `dir`. + * + * Compatible with the #162 data-loss guard: only `isBundleOwnedEntry` + * names are moved aside or replaced — foreign files in `--out` survive. + * + * Re-commit safety: bundle-owned entries are renamed to a sibling + * aside directory before the new artifacts land. The prior `meta.json` + * is moved aside first (§3/§7.3 — no meta ⇒ refuse to consume) so a + * concurrent reader never sees meta pointing at steps already gone. + * The new `meta.json` is installed last. On failure, aside entries are + * restored so a failed re-commit never leaves the directory unusable. + */ +export async function commitBundle( tmpDir: string, dir: string, files: ReadonlyArray, ): Promise { - // (1) Remove the prior bundle's completion signal FIRST. - await unlink(join(dir, 'meta.json')).catch(() => undefined); - - // (2) Sweep stale top-level files that the new bundle won't write. - // If the prior run wrote `video.mp4` and the new run has no video, - // an in-place rename leaves the old video lingering. Only entries the - // bundle format OWNS are candidates: `--out` may point at a directory - // that also holds the user's unrelated files, and those must survive - // the commit (deleting them would be silent data loss). - const topLevel = files.filter(f => !f.startsWith('steps/')); - const newTopLevelSet = new Set(topLevel); - newTopLevelSet.add('meta.json'); // about to land last, do not delete - const existing = await readdir(dir).catch(() => [] as string[]); - for (const entry of existing) { - // Preserve the writer's own scratch dir + the .partial marker - // (we'll re-evaluate .partial at the end of commit). Any other - // bundle-owned entry not-listed in the new bundle is stale. - if (entry === '.tmp' || entry === '.partial') continue; - if (newTopLevelSet.has(entry)) continue; - if (entry === 'steps') continue; // handled below - if (!isBundleOwnedEntry(entry)) continue; // foreign file — never touch - await rm(join(dir, entry), { recursive: true, force: true }); - } + const parent = dirname(dir); + const base = basename(dir); + const asideDir = join(parent, `.${base}.aside.${randomUUID()}`); + const asideLog: Array<{ asidePath: string; restorePath: string }> = []; + + const asideIfPresent = async (entry: string): Promise => { + const restorePath = join(dir, entry); + if (!(await pathExists(restorePath))) return; + const asidePath = join(asideDir, entry); + await mkdir(dirname(asidePath), { recursive: true }); + await rename(restorePath, asidePath); + asideLog.push({ asidePath, restorePath }); + }; - // (3) Replace `/steps/` with `/steps/`. - const stepsTmp = join(tmpDir, 'steps'); - const stepsDir = join(dir, 'steps'); - await rm(stepsDir, { recursive: true, force: true }); - if (await dirExists(stepsTmp)) { - await rename(stepsTmp, stepsDir); - } + const rollback = async (): Promise => { + for (const { asidePath, restorePath } of [...asideLog].reverse()) { + await rm(restorePath, { recursive: true, force: true }).catch(() => undefined); + await rename(asidePath, restorePath).catch(() => undefined); + } + await rm(asideDir, { recursive: true, force: true }).catch(() => undefined); + }; - // (4) Top-level files (result/failure/code/video). meta.json renames - // LAST; track it separately. - const metaIdx = topLevel.indexOf('meta.json'); - const beforeMeta = metaIdx >= 0 ? topLevel.filter((_, i) => i !== metaIdx) : topLevel; - for (const file of beforeMeta) { - await rename(join(tmpDir, file), join(dir, file)); - } + try { + const topLevel = files.filter(f => !f.startsWith('steps/')); + const newTopLevelSet = new Set(topLevel); + newTopLevelSet.add('meta.json'); + + // meta.json first — its presence is the completion signal; do not + // move steps (or anything else) aside while a stale meta is still visible. + await asideIfPresent('meta.json'); + + const existing = await readdir(dir).catch(() => [] as string[]); + for (const entry of existing) { + if (entry === '.tmp' || entry === 'meta.json') continue; + if (!isBundleOwnedEntry(entry)) continue; + + const isStale = entry !== 'steps' && entry !== '.partial' && !newTopLevelSet.has(entry); + const willReplace = entry === 'steps' || newTopLevelSet.has(entry); + if (isStale || willReplace) { + await asideIfPresent(entry); + } + } - // (5) meta.json LAST → atomic completion signal. - if (metaIdx >= 0) { - await rename(join(tmpDir, 'meta.json'), join(dir, 'meta.json')); - } + const stepsTmp = join(tmpDir, 'steps'); + const stepsDir = join(dir, 'steps'); + if (await dirExists(stepsTmp)) { + await rename(stepsTmp, stepsDir); + } - // .partial from a prior aborted run is now stale. Remove it so an - // agent inspecting the dir sees only the fresh bundle. - await unlink(join(dir, '.partial')).catch(() => undefined); + const metaIdx = topLevel.indexOf('meta.json'); + const beforeMeta = metaIdx >= 0 ? topLevel.filter((_, i) => i !== metaIdx) : topLevel; + for (const file of beforeMeta) { + await rename(join(tmpDir, file), join(dir, file)); + } - // Clean up the now-empty tmp dir. - await rm(tmpDir, { recursive: true, force: true }); + if (metaIdx >= 0) { + await rename(join(tmpDir, 'meta.json'), join(dir, 'meta.json')); + } + + await unlink(join(dir, '.partial')).catch(() => undefined); + await rm(tmpDir, { recursive: true, force: true }); + // Best-effort aside cleanup — failures must not roll back a committed bundle. + await rm(asideDir, { recursive: true, force: true }).catch(() => undefined); + } catch (err) { + await rollback(); + await rm(tmpDir, { recursive: true, force: true }).catch(() => undefined); + throw err; + } +} + +async function pathExists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } } async function dirExists(path: string): Promise { From 36e5b14fd1e2983bdacadd895313e0ef5881b9cd Mon Sep 17 00:00:00 2001 From: Lex Date: Sat, 18 Jul 2026 06:53:36 +0700 Subject: [PATCH 067/117] fix(init): debug log whoami fallback (#223) Co-authored-by: Lexiie <28455136+Lexiie@users.noreply.github.com> --- src/commands/init.test.ts | 39 +++++++++++++++++++++++++++++++++++++++ src/commands/init.ts | 6 +++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 617fd6f..0c1bd8e 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -237,6 +237,45 @@ describe('runInit — happy path (interactive)', () => { expect(agent.skills).toContain('testsprite-verify'); expect(agent.skills).toContain('testsprite-onboard'); }); + + it('debug mode reports a display-only whoami lookup failure without corrupting JSON stdout', async () => { + const { captured, deps } = makeCapture(); + let callCount = 0; + const fetchMock = vi.fn(async () => { + callCount += 1; + if (callCount === 1) { + return new Response(JSON.stringify(ME), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'Invalid API key', + nextAction: 'Provide a valid key.', + requestId: 'r-whoami', + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ); + }) as unknown as InitDeps['fetchImpl']; + + await runInit( + makeBaseOpts({ apiKey: 'sk-json-test', debug: true, noAgent: true, output: 'json' }), + { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + }, + ); + + const parsed = JSON.parse(captured.stdout.join('\n')) as Record; + expect(parsed.status).toBe('initialized'); + expect(captured.stderr.some(line => line.includes('setup identity lookup failed'))).toBe(true); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/init.ts b/src/commands/init.ts index c2d5678..6357e3b 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -291,9 +291,13 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise Date: Sat, 18 Jul 2026 06:53:51 +0700 Subject: [PATCH 068/117] fix(auth): tighten Windows credentials ACL (#104) (#253) * fix(auth): tighten Windows credentials ACL * docs(auth): document credential permission tightening --- src/lib/credentials.test.ts | 41 +++++++++++++++++++++++- src/lib/credentials.ts | 64 ++++++++++++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index d50ad52..194c1ff 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -1,7 +1,7 @@ import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DEFAULT_PROFILE, assertValidProfileName, @@ -191,6 +191,45 @@ describe('ensureRestrictiveMode', () => { expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); }); + it('tightens the Windows ACL with icacls instead of POSIX chmod', () => { + mkdirSync(tmpRoot, { recursive: true }); + writeFileSync(credentialsPath, 'data', { mode: 0o666 }); + const spawn = vi.fn(() => ({ status: 0, signal: null, output: [], pid: 123 })) as never; + + ensureRestrictiveMode(credentialsPath, { + platform: 'win32', + env: { USERNAME: 'alice' } as NodeJS.ProcessEnv, + spawnSync: spawn, + }); + + expect(spawn).toHaveBeenCalledWith( + 'icacls', + [credentialsPath, '/inheritance:r', '/grant:r', 'alice:F'], + { + shell: false, + stdio: 'ignore', + windowsHide: true, + }, + ); + }); + + it('warns on Windows when credentials ACL tightening cannot run', () => { + mkdirSync(tmpRoot, { recursive: true }); + writeFileSync(credentialsPath, 'data'); + const warnings: string[] = []; + const spawn = vi.fn(() => ({ status: 0, signal: null, output: [], pid: 123 })) as never; + + ensureRestrictiveMode(credentialsPath, { + platform: 'win32', + env: {} as NodeJS.ProcessEnv, + spawnSync: spawn, + warn: line => warnings.push(line), + }); + + expect(spawn).not.toHaveBeenCalled(); + expect(warnings.join('\n')).toContain('credentials file permissions were not tightened'); + }); + // POSIX-only premise: Windows has no 0644/0600 distinction to downgrade. it.skipIf(process.platform === 'win32')('downgrades over-permissive modes', () => { mkdirSync(tmpRoot, { recursive: true }); diff --git a/src/lib/credentials.ts b/src/lib/credentials.ts index 21e2ac0..6c20b1f 100644 --- a/src/lib/credentials.ts +++ b/src/lib/credentials.ts @@ -7,6 +7,7 @@ import { statSync, writeFileSync, } from 'node:fs'; +import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { localValidationError } from './errors.js'; @@ -61,6 +62,17 @@ export interface CredentialsOptions { path?: string; } +interface RestrictiveModeOptions { + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; + spawnSync?: ( + command: string, + args: readonly string[], + options: { shell: false; stdio: 'ignore'; windowsHide: true }, + ) => SpawnSyncReturns; + warn?: (line: string) => void; +} + const FILE_KEY_TO_FIELD: Record = { api_key: 'apiKey', api_url: 'apiUrl', @@ -172,12 +184,62 @@ export function deleteProfile(profile: string, options: CredentialsOptions = {}) return true; } -export function ensureRestrictiveMode(path: string): void { +/** + * Enforce restrictive access on the credentials file after atomic writes. + * POSIX hosts use chmod(0600); Windows hosts use ACL tightening via icacls. + */ +export function ensureRestrictiveMode(path: string, options: RestrictiveModeOptions = {}): void { if (!existsSync(path)) return; + if ((options.platform ?? process.platform) === 'win32') { + ensureWindowsRestrictiveAcl(path, options); + return; + } const overpermissive = (statSync(path).mode & 0o077) !== 0; if (overpermissive) chmodSync(path, 0o600); } +/** + * Restrict a Windows credentials file to the current user using icacls. + * The command is invoked with an args array so credential paths are never shell-interpreted. + */ +function ensureWindowsRestrictiveAcl(path: string, options: RestrictiveModeOptions): void { + const username = (options.env ?? process.env).USERNAME?.trim(); + if (!username) { + warnWindowsAcl( + 'could not determine the Windows username; credentials file permissions were not tightened', + options, + ); + return; + } + + const run = options.spawnSync ?? spawnSync; + const result = run('icacls', [path, '/inheritance:r', '/grant:r', `${username}:F`], { + shell: false, + stdio: 'ignore', + windowsHide: true, + }); + + if (result.error) { + warnWindowsAcl( + `icacls failed while tightening credentials file permissions: ${result.error.message}`, + options, + ); + return; + } + if (result.status !== 0) { + warnWindowsAcl( + `icacls exited with status ${result.status ?? 'unknown'}; credentials file permissions may be too broad`, + options, + ); + } +} + +/** Emit an explicit warning when Windows ACL tightening cannot be completed. */ +function warnWindowsAcl(message: string, options: RestrictiveModeOptions): void { + const warn = options.warn ?? ((line: string) => process.stderr.write(`${line}\n`)); + warn(`[warning] ${message}`); +} + function resolvePath(options: CredentialsOptions): string { return options.path ?? defaultCredentialsPath(); } From 34f5d430797437ab2f67910fe356c3732ea0be57 Mon Sep 17 00:00:00 2001 From: nopp Date: Sat, 18 Jul 2026 06:54:04 +0700 Subject: [PATCH 069/117] fix(http): avoid retrying keyless writes (#256) --- src/lib/http.test.ts | 46 ++++++++++++++++++++++++++++++++++++++++++++ src/lib/http.ts | 23 ++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 2baff2c..e8f4c35 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -328,6 +328,31 @@ describe('HttpClient error mapping', () => { expect(fetchImpl).toHaveBeenCalledTimes(4); }); + it('does not retry transport errors for keyless writes', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('ECONNRESET'); + }); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.post('/projects', { body: { name: 'Checkout' } })).rejects.toBeInstanceOf( + TransportError, + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('retries transport errors for writes with an idempotency key', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('ECONNRESET'); + }); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect( + client.post('/projects', { + body: { name: 'Checkout' }, + headers: { 'Idempotency-Key': 'op_123' }, + }), + ).rejects.toBeInstanceOf(TransportError); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + it('does not retry AbortError', async () => { const fetchImpl = vi.fn(async () => { const err = new Error('aborted'); @@ -399,6 +424,27 @@ describe('HttpClient transport-edge statuses', () => { }, ); + it('does not retry bare transport-edge responses for keyless writes', async () => { + const fetchImpl = vi.fn(async () => new Response('proxy gateway html', { status: 502 })); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.post('/projects', { body: { name: 'Checkout' } })).rejects.toBeInstanceOf( + TransportError, + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('retries bare transport-edge responses for writes with an idempotency key', async () => { + const fetchImpl = vi.fn(async () => new Response('proxy gateway html', { status: 502 })); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect( + client.post('/projects', { + body: { name: 'Checkout' }, + headers: { 'idempotency-key': 'op_123' }, + }), + ).rejects.toBeInstanceOf(TransportError); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + it('502 carrying our envelope still maps to its catalog code', async () => { const body = { error: { diff --git a/src/lib/http.ts b/src/lib/http.ts index f0139ca..21369de 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -487,6 +487,7 @@ export class HttpClient { const url = buildUrl(this.baseUrl, path, options.query); const requestId = options.requestId ?? newRequestId(); + const allowTransportRetry = canRetryTransport(method, options); let attempt = 0; while (true) { @@ -557,7 +558,9 @@ export class HttpClient { errorCode: 'TRANSPORT', durationMs: Date.now() - startedAt, }); - const decision = transportRetryDecision(attempt, this.random); + const decision = allowTransportRetry + ? transportRetryDecision(attempt, this.random) + : { retry: false, delayMs: 0 }; if (!decision.retry) throw new TransportError(message, requestId); this.transition( `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, @@ -635,7 +638,9 @@ export class HttpClient { errorCode: 'TRANSPORT', durationMs, }); - const decision = transportRetryDecision(attempt, this.random); + const decision = allowTransportRetry + ? transportRetryDecision(attempt, this.random) + : { retry: false, delayMs: 0 }; if (!decision.retry) { throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); } @@ -945,6 +950,20 @@ function transportRetryDecision(attempt: number, random: () => number): RetryDec return { retry: true, delayMs: backoffDelay(attempt, random) }; } +function canRetryTransport(method: string, options: RequestOptions): boolean { + return isIdempotentMethod(method) || hasIdempotencyKey(options.headers); +} + +function isIdempotentMethod(method: string): boolean { + const normalized = method.toUpperCase(); + return normalized === 'GET' || normalized === 'HEAD'; +} + +function hasIdempotencyKey(headers: Record | undefined): boolean { + if (!headers) return false; + return Object.keys(headers).some(name => name.toLowerCase() === 'idempotency-key'); +} + function apiRetryDecision( code: ErrorCode, attempt: number, From 759a1c7df81b926c70ab125b16754460e47055dd Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:54:18 -0400 Subject: [PATCH 070/117] feat(cli): add "testsprite completion" for bash/zsh/fish (#227) Emit a shell completion script for bash, zsh, or fish. Command names, subcommands, and global flags are derived from the fully-assembled Commander tree at call time (buildCompletionSpec walks program.commands), so the script can never drift from the real command surface. The shell auto-detects from $SHELL when the argument is omitted. Fixes #74 --- src/commands/completion.test.ts | 91 ++++++++++ src/commands/completion.ts | 164 ++++++++++++++++++ src/index.ts | 23 +++ test/__snapshots__/help.snapshot.test.ts.snap | 1 + 4 files changed, 279 insertions(+) create mode 100644 src/commands/completion.test.ts create mode 100644 src/commands/completion.ts diff --git a/src/commands/completion.test.ts b/src/commands/completion.test.ts new file mode 100644 index 0000000..8388a09 --- /dev/null +++ b/src/commands/completion.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import type { CompletionSpec } from './completion.js'; +import { createCompletionCommand, detectShell, isShell, renderCompletion } from './completion.js'; + +const SPEC: CompletionSpec = { + program: 'testsprite', + commands: ['setup', 'auth', 'test', 'doctor', 'completion', 'help'], + subcommands: { auth: ['status', 'remove'], test: ['run', 'wait'] }, + globalFlags: ['--output', '--profile', '--help'], +}; + +describe('isShell / detectShell', () => { + it('recognizes the three supported shells', () => { + expect(isShell('bash')).toBe(true); + expect(isShell('zsh')).toBe(true); + expect(isShell('fish')).toBe(true); + expect(isShell('powershell')).toBe(false); + }); + + it('detects the shell from a $SHELL path', () => { + expect(detectShell({ SHELL: '/bin/bash' })).toBe('bash'); + expect(detectShell({ SHELL: '/usr/bin/zsh' })).toBe('zsh'); + expect(detectShell({ SHELL: '/usr/local/bin/fish' })).toBe('fish'); + }); + + it('returns undefined for an unknown or missing shell', () => { + expect(detectShell({ SHELL: '/bin/sh' })).toBeUndefined(); + expect(detectShell({})).toBeUndefined(); + }); +}); + +describe('renderCompletion', () => { + it('bash script wires a completion function and lists commands, subcommands, flags', () => { + const script = renderCompletion('bash', SPEC); + expect(script).toContain('complete -F _testsprite_completion testsprite'); + expect(script).toContain('setup'); + expect(script).toContain('auth) COMPREPLY'); + expect(script).toContain('status remove'); + expect(script).toContain('--output'); + }); + + it('zsh script declares #compdef and per-group subcommands', () => { + const script = renderCompletion('zsh', SPEC); + expect(script.startsWith('#compdef testsprite')).toBe(true); + expect(script).toContain('compdef _testsprite testsprite'); + expect(script).toContain('run wait'); + }); + + it('fish script uses complete -c with subcommand conditions and flags', () => { + const script = renderCompletion('fish', SPEC); + expect(script).toContain('complete -c testsprite -f'); + expect(script).toContain('__fish_seen_subcommand_from auth'); + expect(script).toContain('-l output'); + }); +}); + +describe('createCompletionCommand', () => { + function run(args: string[], env: NodeJS.ProcessEnv): Promise { + const out: string[] = []; + const cmd = createCompletionCommand(() => SPEC, { env, stdout: line => out.push(line) }); + return cmd.parseAsync(args, { from: 'user' }).then(() => out); + } + + it('prints the requested shell script from an explicit argument', async () => { + const out = await run(['bash'], {}); + expect(out.join('\n')).toContain('complete -F'); + }); + + it('auto-detects the shell from $SHELL when no argument is given', async () => { + const out = await run([], { SHELL: '/usr/bin/zsh' }); + expect(out.join('\n')).toContain('#compdef testsprite'); + }); + + it('rejects an unsupported shell with VALIDATION_ERROR (exit 5)', async () => { + const cmd = createCompletionCommand(() => SPEC, { env: {}, stdout: () => undefined }); + await expect(cmd.parseAsync(['powershell'], { from: 'user' })).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + }); + }); + + it('errors when the shell cannot be detected and none is given', async () => { + const cmd = createCompletionCommand(() => SPEC, { env: {}, stdout: () => undefined }); + await expect(cmd.parseAsync([], { from: 'user' })).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + }); + }); + + it('is named "completion"', () => { + expect(createCompletionCommand(() => SPEC).name()).toBe('completion'); + }); +}); diff --git a/src/commands/completion.ts b/src/commands/completion.ts new file mode 100644 index 0000000..15cf7a4 --- /dev/null +++ b/src/commands/completion.ts @@ -0,0 +1,164 @@ +/** + * `testsprite completion [bash|zsh|fish]` — emit a shell completion script. + * + * The command names, per-group subcommands, and global flags are NOT hardcoded: + * `index.ts` builds a {@link CompletionSpec} by walking the fully-assembled + * Commander program and passes it in, so the generated script can never drift + * from the real command tree. `renderCompletion` is a pure function of the spec, + * which keeps it unit-testable without a live program. + * + * Usage: + * bash: eval "$(testsprite completion bash)" (add to ~/.bashrc) + * zsh: testsprite completion zsh > ~/.zsh/_testsprite (on your fpath) + * fish: testsprite completion fish | source (add to config.fish) + */ + +import { Command } from 'commander'; +import { localValidationError } from '../lib/errors.js'; + +export const SUPPORTED_SHELLS = ['bash', 'zsh', 'fish'] as const; +export type Shell = (typeof SUPPORTED_SHELLS)[number]; + +export interface CompletionSpec { + /** Binary name, e.g. "testsprite". */ + program: string; + /** Top-level command names. */ + commands: string[]; + /** command name -> its subcommand names (only groups that have subcommands). */ + subcommands: Record; + /** Global long option flags (e.g. "--output"). */ + globalFlags: string[]; +} + +export interface CompletionDeps { + env?: NodeJS.ProcessEnv; + stdout?: (line: string) => void; +} + +export function isShell(value: string): value is Shell { + return (SUPPORTED_SHELLS as readonly string[]).includes(value); +} + +/** Best-effort shell detection from `$SHELL` (e.g. "/bin/zsh" -> "zsh"). */ +export function detectShell(env: NodeJS.ProcessEnv): Shell | undefined { + const shellPath = env.SHELL ?? ''; + const base = shellPath.slice(shellPath.lastIndexOf('/') + 1); + return isShell(base) ? base : undefined; +} + +export function renderCompletion(shell: Shell, spec: CompletionSpec): string { + switch (shell) { + case 'bash': + return renderBash(spec); + case 'zsh': + return renderZsh(spec); + case 'fish': + return renderFish(spec); + } +} + +function renderBash(spec: CompletionSpec): string { + const fn = `_${spec.program}_completion`; + const lines = [ + `# ${spec.program} bash completion. Enable with: eval "$(${spec.program} completion bash)"`, + `${fn}() {`, + ' local cur prev', + ' cur="${COMP_WORDS[COMP_CWORD]}"', + ' prev="${COMP_WORDS[COMP_CWORD-1]}"', + ` local commands="${spec.commands.join(' ')}"`, + ` local global_flags="${spec.globalFlags.join(' ')}"`, + ' case "$prev" in', + ...Object.entries(spec.subcommands).map( + ([group, subs]) => + ` ${group}) COMPREPLY=( $(compgen -W "${subs.join(' ')}" -- "$cur") ); return;;`, + ), + ' esac', + ' if [[ "$cur" == -* ]]; then', + ' COMPREPLY=( $(compgen -W "$global_flags" -- "$cur") ); return', + ' fi', + ' COMPREPLY=( $(compgen -W "$commands" -- "$cur") )', + '}', + `complete -F ${fn} ${spec.program}`, + ]; + return lines.join('\n'); +} + +function renderZsh(spec: CompletionSpec): string { + const fn = `_${spec.program}`; + const lines = [ + `#compdef ${spec.program}`, + `# ${spec.program} zsh completion. Enable with: ${spec.program} completion zsh > "$fpath[1]/_${spec.program}"`, + `${fn}() {`, + ' local -a commands', + ` commands=(${spec.commands.join(' ')})`, + ' if (( CURRENT == 2 )); then', + " _describe 'command' commands", + ' return', + ' fi', + ' case "${words[2]}" in', + ...Object.entries(spec.subcommands).map( + ([group, subs]) => + ` ${group}) local -a subs; subs=(${subs.join(' ')}); _describe 'subcommand' subs;;`, + ), + ' esac', + '}', + `compdef ${fn} ${spec.program}`, + ]; + return lines.join('\n'); +} + +function renderFish(spec: CompletionSpec): string { + const lines = [ + `# ${spec.program} fish completion. Enable with: ${spec.program} completion fish | source`, + `complete -c ${spec.program} -f`, + ...spec.commands.map( + command => `complete -c ${spec.program} -n '__fish_use_subcommand' -a '${command}'`, + ), + ...Object.entries(spec.subcommands).flatMap(([group, subs]) => + subs.map( + sub => `complete -c ${spec.program} -n '__fish_seen_subcommand_from ${group}' -a '${sub}'`, + ), + ), + ...spec.globalFlags.map(flag => `complete -c ${spec.program} -l ${flag.replace(/^--/, '')}`), + ]; + return lines.join('\n'); +} + +export function createCompletionCommand( + getSpec: () => CompletionSpec, + deps: CompletionDeps = {}, +): Command { + return new Command('completion') + .description('Print a shell completion script (bash|zsh|fish)') + .argument( + '[shell]', + 'Shell to generate for (bash|zsh|fish); auto-detected from $SHELL when omitted', + ) + .addHelpText( + 'after', + '\nExamples:\n' + + ' eval "$(testsprite completion bash)" # bash, current session\n' + + ' testsprite completion zsh > ~/.zsh/_testsprite\n' + + ' testsprite completion fish | source # fish, current session', + ) + .action((shellArg: string | undefined, _cmdOpts: unknown) => { + const env = deps.env ?? process.env; + const shell = shellArg ?? detectShell(env); + if (shell === undefined) { + throw localValidationError( + 'shell', + `could not detect the shell from $SHELL; pass one explicitly (${SUPPORTED_SHELLS.join(', ')})`, + [...SUPPORTED_SHELLS], + ); + } + if (!isShell(shell)) { + throw localValidationError( + 'shell', + `unsupported shell "${shell}"; use one of: ${SUPPORTED_SHELLS.join(', ')}`, + [...SUPPORTED_SHELLS], + ); + } + const write = deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)); + write(renderCompletion(shell, getSpec())); + }); +} diff --git a/src/index.ts b/src/index.ts index ae3f944..31d3399 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; +import { createCompletionCommand, type CompletionSpec } from './commands/completion.js'; import { createDoctorCommand } from './commands/doctor.js'; import { createDeprecatedInitCommand, @@ -92,6 +93,28 @@ program.addCommand(createTestCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); program.addCommand(createDoctorCommand()); +program.addCommand(createCompletionCommand(() => buildCompletionSpec())); + +// Derive the shell-completion spec from the fully-assembled command tree at call +// time (not module-load), so `testsprite completion` can never drift from the +// real commands, subcommands, and global flags. +function buildCompletionSpec(): CompletionSpec { + const subcommands: Record = {}; + for (const command of program.commands) { + const subs = command.commands.map(sub => sub.name()).filter(name => name !== 'help'); + if (subs.length > 0) subcommands[command.name()] = subs; + } + const flags = program.options + .map(option => option.long) + .filter((long): long is string => typeof long === 'string'); + if (!flags.includes('--help')) flags.push('--help'); + return { + program: 'testsprite', + commands: [...new Set([...program.commands.map(command => command.name()), 'help'])], + subcommands, + globalFlags: flags, + }; +} // Buffer Commander error messages instead of writing immediately. The catch // block re-emits in the correct format (JSON or text) once the requested diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index b0681c5..6f3e290 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -749,6 +749,7 @@ Commands: (proactive pre-flight before a large test run) doctor Diagnose CLI setup: version, Node, profile, endpoint, credentials, connectivity, skill + completion [shell] Print a shell completion script (bash|zsh|fish) help [command] display help for command " `; From 45dcd0895f10b406646333e537c658f2b3cd0b41 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:54:29 -0500 Subject: [PATCH 071/117] fix(agent): update verify skill auth commands (#216) --- docs/cli-v1-agent-install/skill-template.md | 6 +++--- skills/testsprite-verify.codex.md | 4 ++-- skills/testsprite-verify.skill.md | 6 +++--- src/lib/agent-targets.test.ts | 21 +++++++++++++++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/cli-v1-agent-install/skill-template.md b/docs/cli-v1-agent-install/skill-template.md index d051a39..764e04e 100644 --- a/docs/cli-v1-agent-install/skill-template.md +++ b/docs/cli-v1-agent-install/skill-template.md @@ -61,13 +61,13 @@ because . Treat this as unverified until that's resolved." Don't claim done. ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials configured? +testsprite auth status # credentials configured? ``` - `--version` fails → the CLI isn't installed. Tell the user to install the TestSprite CLI (see the TestSprite docs) and stop; don't install it for them. -- `auth whoami` fails → no credentials. Tell the user they can run - `testsprite auth configure`, then stop. +- `auth status` fails → no credentials. Tell the user they can run + `testsprite setup`, then stop. ## 2. Find the project diff --git a/skills/testsprite-verify.codex.md b/skills/testsprite-verify.codex.md index 5d28f16..d3c5373 100644 --- a/skills/testsprite-verify.codex.md +++ b/skills/testsprite-verify.codex.md @@ -22,11 +22,11 @@ unverified-because-undeployed and stop. If the user explicitly named a tool ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials valid? +testsprite auth status # credentials valid? ``` If `--version` fails, tell the user to install the CLI and stop. -If `auth whoami` fails, tell the user to run `testsprite auth configure` and stop. +If `auth status` fails, tell the user to run `testsprite setup` and stop. ### 2. Find the project diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index 5020e42..40d60f2 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -67,13 +67,13 @@ because . Treat this as unverified until that's resolved." Don't claim done. ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials configured? +testsprite auth status # credentials configured? ``` - `--version` fails → the CLI isn't installed. Tell the user to install the TestSprite CLI (see the TestSprite docs) and stop; don't install it for them. -- `auth whoami` fails → no credentials. Tell the user they can run - `testsprite auth configure`, then stop. +- `auth status` fails → no credentials. Tell the user they can run + `testsprite setup`, then stop. ## 2. Find the project diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 2ac9158..73feee3 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -449,6 +449,27 @@ describe('loadCodexSkillBody', () => { }); }); +// --------------------------------------------------------------------------- +// content integrity — current auth command names +// --------------------------------------------------------------------------- + +describe('content integrity — testsprite-verify auth commands', () => { + it('uses current auth/setup commands in every canonical verify skill asset', () => { + const assets = [ + ['docs template', templateRaw], + ['own-file skill body', loadSkillBodyFor('testsprite-verify')], + ['codex skill body', codexContentFor('testsprite-verify')], + ] as const; + + for (const [name, content] of assets) { + expect(content, name).toContain('testsprite auth status'); + expect(content, name).toContain('testsprite setup'); + expect(content, name).not.toContain('auth whoami'); + expect(content, name).not.toContain('auth configure'); + } + }); +}); + // --------------------------------------------------------------------------- // MANAGED_SECTION sentinels // --------------------------------------------------------------------------- From eaf0585dd29b960161fe3d160adb23ce0e6040ad Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:00:05 -0500 Subject: [PATCH 072/117] fix(flaky): propagate auth trigger errors (#217) --- src/commands/test.flaky.spec.ts | 42 +++++++++++++++++++++++++++++++++ src/commands/test.ts | 15 ++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/commands/test.flaky.spec.ts b/src/commands/test.flaky.spec.ts index 0c4450e..8076f3e 100644 --- a/src/commands/test.flaky.spec.ts +++ b/src/commands/test.flaky.spec.ts @@ -18,6 +18,7 @@ import { runFlaky } from './test.js'; type FetchInput = Parameters[0]; type RunStatus = 'passed' | 'failed' | 'blocked' | 'cancelled'; +type TriggerAuthErrorCode = 'AUTH_REQUIRED' | 'AUTH_INVALID' | 'AUTH_FORBIDDEN'; function urlOf(input: FetchInput): string { return typeof input === 'string' @@ -46,6 +47,7 @@ function makeFlakyFetch(opts: { statuses: RunStatus[]; testType?: 'frontend' | 'backend'; notFoundOnTrigger?: boolean; + triggerAuthError?: TriggerAuthErrorCode; }): { fetchImpl: FetchImpl; triggerCount: () => number } { let triggers = 0; const testType = opts.testType ?? 'frontend'; @@ -67,6 +69,17 @@ function makeFlakyFetch(opts: { } if (method === 'POST' && url.includes('/runs/rerun')) { + if (opts.triggerAuthError) { + return jsonResponse(opts.triggerAuthError === 'AUTH_FORBIDDEN' ? 403 : 401, { + error: { + code: opts.triggerAuthError, + message: 'auth failed', + nextAction: 'run setup', + requestId: 'req_auth', + details: {}, + }, + }); + } if (opts.notFoundOnTrigger) { return jsonResponse(404, { error: { @@ -345,6 +358,35 @@ describe('runFlaky', () => { expect((err as ApiError).code).toBe('NOT_FOUND'); }); + it.each(['AUTH_REQUIRED', 'AUTH_INVALID', 'AUTH_FORBIDDEN'] as const)( + 'propagates %s during trigger instead of scoring an error attempt', + async code => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: [], + triggerAuthError: code, + }); + const { deps, stdout } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect(err).toMatchObject({ code, exitCode: 3 }); + expect(stdout).toEqual([]); + expect(triggerCount()).toBe(0); + }, + ); + it('rejects --runs below the range (0) with a validation error (exit 5)', async () => { const { fetchImpl } = makeFlakyFetch({ statuses: [] }); const { deps } = makeDeps(fetchImpl); diff --git a/src/commands/test.ts b/src/commands/test.ts index 837c170..e82c5f4 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -9379,6 +9379,18 @@ const MAX_FLAKY_RUNS = 10; /** Default replay count when `--runs` is omitted. */ const DEFAULT_FLAKY_RUNS = 5; +function isFlakyFatalTriggerError(err: unknown): boolean { + if (!(err instanceof ApiError)) return false; + switch (err.code) { + case 'AUTH_REQUIRED': + case 'AUTH_INVALID': + case 'AUTH_FORBIDDEN': + return true; + default: + return false; + } +} + interface RunTestFlakyOptions extends CommonOptions { testId: string; /** Number of replays to run (1..MAX_FLAKY_RUNS). */ @@ -9477,6 +9489,9 @@ export async function runFlaky( }, }); } + if (isFlakyFatalTriggerError(err)) { + throw err; + } // Any other trigger error is recorded as an errored attempt so a single // transient blip doesn't abort a long stability probe. const code = err instanceof ApiError ? err.code : 'ERROR'; From 8db1882964b93db3dcd9b7de526cc7d179cd6619 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:00:18 -0400 Subject: [PATCH 073/117] feat(test): add "test open " to jump from the terminal to the dashboard (#226) * feat(test): add "test open " to jump from the terminal to the dashboard * fix(open): derive the dry-run URL via resolvePortalUrl and handle async spawn ENOENT * fix(browser): map non-http(s) URL rejection to exit 5 (validation error) openInBrowser threw a plain Error for a non-http(s) URL, which fell through the top-level handler and exited 1. Route it through localValidationError so it is classified as VALIDATION_ERROR and maps to the documented exit code 5, matching every other bad-argument path. Test asserts the exit code, not the message string. --- src/commands/test.test.ts | 90 ++++++++++++++++ src/commands/test.ts | 80 ++++++++++++++ src/lib/browser.test.ts | 100 ++++++++++++++++++ src/lib/browser.ts | 63 +++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 4 + 5 files changed, 337 insertions(+) create mode 100644 src/lib/browser.test.ts create mode 100644 src/lib/browser.ts diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 9493908..bd238d9 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -33,6 +33,7 @@ import { runGet, runLint, runList, + runOpen, runPlanPut, runResult, runScaffold, @@ -132,6 +133,7 @@ describe('createTestCommand — surface', () => { 'get', 'lint', 'list', + 'open', 'plan', 'rerun', 'result', @@ -2450,6 +2452,94 @@ describe('runScaffold', () => { }); }); +describe('runOpen', () => { + // The mock endpoint host has no portal mapping; the operator override is the + // supported escape hatch and gives the tests a deterministic base. + beforeEach(() => { + process.env.TESTSPRITE_PORTAL_URL = 'https://portal.example.com'; + }); + afterEach(() => { + delete process.env.TESTSPRITE_PORTAL_URL; + }); + + const TEST_ROW = { + id: 'test_open_me', + projectId: 'project_alice', + projectName: 'Alice', + name: 'Checkout', + type: 'frontend', + createdFrom: 'cli', + status: 'ready', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }; + + it('prints the dashboard URL and spawns the opener with it', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const out: string[] = []; + const opened: string[] = []; + const result = await runOpen( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_open_me', + noBrowser: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + url => opened.push(url), + ); + expect(result.dashboardUrl).toContain('/dashboard/tests/project_alice/test/test_open_me'); + expect(out.join('\n')).toContain(result.dashboardUrl); + expect(opened).toEqual([result.dashboardUrl]); + }); + + it('--no-browser prints the URL but never spawns', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const out: string[] = []; + const opened: string[] = []; + await runOpen( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_open_me', + noBrowser: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + url => opened.push(url), + ); + expect(opened).toEqual([]); + expect((JSON.parse(out.join('')) as { dashboardUrl: string }).dashboardUrl).toContain( + 'test_open_me', + ); + }); + + it('a broken opener downgrades to a stderr hint, never a failure', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const errs: string[] = []; + await expect( + runOpen( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_open_me', + noBrowser: false, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) }, + () => { + throw new Error('no display'); + }, + ), + ).resolves.toBeDefined(); + expect(errs.join('\n')).toContain('could not launch a browser'); + }); +}); + describe('runSteps', () => { it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index e82c5f4..4a594dd 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -10,6 +10,7 @@ import { rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path'; import { randomUUID } from 'node:crypto'; import { Command } from 'commander'; +import { openInBrowser } from '../lib/browser.js'; import { emitDryRunBanner, makeHttpClient, @@ -4276,6 +4277,67 @@ export async function runScaffold( return payload; } +export interface OpenOptions extends CommonOptions { + testId: string; + /** Print the URL only; never spawn a browser (SSH/headless/CI/agents). */ + noBrowser: boolean; +} + +/** + * `test open ` (issue #121): jump from the terminal to the test's + * dashboard page. The CLI already computes this deep-link and prints it as + * text on other commands; this closes the last inch (the `gh browse` / + * `cypress open` hop). The URL is ALWAYS printed to stdout (so `--no-browser` + * and headless use still compose), then the OS browser is spawned unless + * --no-browser. An endpoint with no known portal mapping is a hard error + * rather than a silent no-op. + */ +export async function runOpen( + opts: OpenOptions, + deps: TestDeps = {}, + opener: (url: string) => void = openInBrowser, +): Promise<{ dashboardUrl: string }> { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + // Derive the sample through the SAME resolver as the live path (against + // the canonical prod endpoint and the dry-run project id) so the two can + // never drift; the ?? arm is unreachable for the prod mapping but keeps + // the type total. + const sample = { + dashboardUrl: + resolvePortalUrl('https://api.testsprite.com', 'p_dryrun_2026', opts.testId) ?? + `https://www.testsprite.com/dashboard/tests/p_dryrun_2026/test/${encodeURIComponent(opts.testId)}`, + }; + out.print(sample, data => (data as { dashboardUrl: string }).dashboardUrl); + return sample; + } + + const client = makeClient(opts, deps); + // The deep-link needs the projectId; the test record is the source of truth. + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + const dashboardUrl = resolvePortalUrl(resolveApiUrl(opts, deps), test.projectId, opts.testId); + if (dashboardUrl === undefined) { + throw new CLIError( + `no dashboard mapping for this API endpoint; set TESTSPRITE_PORTAL_URL to your Portal origin`, + 1, + ); + } + out.print({ dashboardUrl }, () => dashboardUrl); + if (!opts.noBrowser) { + try { + opener(dashboardUrl); + } catch { + // The URL is already on stdout; a missing opener (containers, minimal + // hosts) downgrades to "open it yourself" instead of a hard failure. + stderrFn('could not launch a browser; open the URL above manually (or use --no-browser)'); + } + } + return { dashboardUrl }; +} + export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -8720,6 +8782,24 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('open ') + .description( + 'Open the test in the TestSprite dashboard: prints the deep-link URL, then spawns your default browser unless --no-browser.', + ) + .option('--no-browser', 'print the URL only (SSH, headless, CI, agents)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, cmdOpts: { browser?: boolean }, command: Command) => { + await runOpen( + { + ...resolveCommonOptions(command), + testId, + noBrowser: cmdOpts.browser === false, + }, + deps, + ); + }); + test .command('steps ') .description( diff --git a/src/lib/browser.test.ts b/src/lib/browser.test.ts new file mode 100644 index 0000000..3d6717b --- /dev/null +++ b/src/lib/browser.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Mock the real spawner so the default `exec` path (detached spawn + unref) +// is exercised without launching a real process. +const spawnMock = vi.fn(); +vi.mock('node:child_process', () => ({ + spawn: (command: string, args: readonly string[], opts: unknown) => + spawnMock(command, args, opts), +})); + +import { openInBrowser } from './browser.js'; +import { ApiError } from './errors.js'; + +describe('openInBrowser', () => { + const url = 'https://portal.example.com/tests/t_123'; + + it('uses `open ` on darwin', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'darwin', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'open', args: [url] }]); + }); + + it('uses rundll32 FileProtocolHandler on win32', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'win32', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'rundll32', args: ['url.dll,FileProtocolHandler', url] }]); + }); + + it('uses xdg-open on other platforms', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'linux', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'xdg-open', args: [url] }]); + }); + + it('refuses a non-http(s) URL with exit 5 before spawning', () => { + const exec = vi.fn(); + let error: unknown; + try { + openInBrowser('file:///etc/passwd', { platform: 'linux', exec }); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiError); + expect((error as ApiError).exitCode).toBe(5); + expect(exec).not.toHaveBeenCalled(); + }); + + it('throws on a malformed URL', () => { + const exec = vi.fn(); + expect(() => openInBrowser('not a url', { exec })).toThrow(); + expect(exec).not.toHaveBeenCalled(); + }); + + describe('default spawner', () => { + beforeEach(() => { + spawnMock.mockReset(); + }); + + it('spawns detached, ignores stdio, and unrefs the child', () => { + const unref = vi.fn(); + spawnMock.mockReturnValue({ unref, on: vi.fn() }); + openInBrowser(url, { platform: 'darwin' }); + expect(spawnMock).toHaveBeenCalledWith('open', [url], { detached: true, stdio: 'ignore' }); + expect(unref).toHaveBeenCalledTimes(1); + }); + + it("handles the child's async 'error' (missing binary) with a stderr hint, not a crash", () => { + // spawn() reports ENOENT asynchronously on the child; an unhandled + // 'error' event would crash the CLI. The default spawner must register + // a listener that degrades to the manual-open hint. + const listeners = new Map void>(); + spawnMock.mockReturnValue({ + unref: vi.fn(), + on: (event: string, listener: (err: Error) => void) => { + listeners.set(event, listener); + }, + }); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + try { + openInBrowser(url, { platform: 'linux' }); + const onError = listeners.get('error'); + expect(onError).toBeDefined(); + // Firing the listener must not throw and must print the hint. + expect(() => onError!(new Error('spawn xdg-open ENOENT'))).not.toThrow(); + expect(String(stderrSpy.mock.calls.at(-1)?.[0])).toContain('could not launch a browser'); + } finally { + stderrSpy.mockRestore(); + } + }); + }); +}); diff --git a/src/lib/browser.ts b/src/lib/browser.ts new file mode 100644 index 0000000..ccd2b8a --- /dev/null +++ b/src/lib/browser.ts @@ -0,0 +1,63 @@ +/** + * Cross-platform "open this URL in the default browser" helper for + * `test open` (issue #121). Spawns the platform opener with an argv array + * (never a shell string) so a URL can never be shell-injected, and refuses + * anything that is not http(s) before any process is spawned. + * + * Platform openers: + * darwin open + * win32 rundll32 url.dll,FileProtocolHandler (avoids `cmd /c start`, + * whose re-parsing would mangle `&` and other metachars in the URL) + * other xdg-open + * + * The child is detached and unref'd so the CLI exits immediately; failures + * are the caller's to surface (it already printed the URL as the fallback). + */ +import { spawn } from 'node:child_process'; +import { localValidationError } from './errors.js'; + +export interface OpenInBrowserDeps { + platform?: NodeJS.Platform; + /** Process spawner taking an argv array. Defaults to a detached spawn. */ + exec?: (command: string, args: readonly string[]) => void; +} + +export function openInBrowser(url: string, deps: OpenInBrowserDeps = {}): void { + const parsed = new URL(url); + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { + // User-input error, not an internal failure: classify as VALIDATION_ERROR + // so it maps to exit 5 (like every other bad-argument path), not exit 1. + throw localValidationError( + 'url', + `must be an http(s) URL (got ${parsed.protocol})`, + undefined, + 'field', + ); + } + const platform = deps.platform ?? process.platform; + const exec = + deps.exec ?? + ((command: string, args: readonly string[]) => { + const child = spawn(command, [...args], { detached: true, stdio: 'ignore' }); + // spawn() reports a missing binary (ENOENT) ASYNCHRONOUSLY on the child, + // so the caller's try/catch cannot see it — and an unhandled 'error' + // event would crash the whole CLI. The URL is already on stdout, so + // degrade to the same manual-open hint the sync failure path prints. + child.on('error', () => { + process.stderr.write( + 'could not launch a browser; open the URL above manually (or use --no-browser)\n', + ); + }); + child.unref(); + }); + + if (platform === 'darwin') { + exec('open', [url]); + return; + } + if (platform === 'win32') { + exec('rundll32', ['url.dll,FileProtocolHandler', url]); + return; + } + exec('xdg-open', [url]); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6f3e290..441e700 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -222,6 +222,10 @@ Commands: definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials. + open [options] Open the test in the TestSprite + dashboard: prints the deep-link URL, + then spawns your default browser unless + --no-browser. steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) From 9d4af09af285ad9863c9c1bc9106d968d3d1bf05 Mon Sep 17 00:00:00 2001 From: nopp Date: Sat, 18 Jul 2026 07:01:37 +0700 Subject: [PATCH 074/117] feat(cli): add text list column controls (#165) (#252) * feat(cli): add text list column controls * fix(cli): address list column review feedback --- src/commands/project.test.ts | 68 +++++++ src/commands/project.ts | 79 ++++---- src/commands/test.result.history.spec.ts | 126 ++++++++++++ src/commands/test.test.ts | 71 +++++++ src/commands/test.ts | 188 +++++++++--------- src/lib/text-table.ts | 94 +++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 8 + 7 files changed, 504 insertions(+), 130 deletions(-) create mode 100644 src/lib/text-table.ts diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 0f8c51c..ae98070 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -89,6 +89,8 @@ describe('createProjectCommand', () => { expect(flagNames).toContain('--page-size'); expect(flagNames).toContain('--starting-token'); expect(flagNames).toContain('--max-items'); + expect(flagNames).toContain('--columns'); + expect(flagNames).toContain('--no-header'); }); }); @@ -285,6 +287,72 @@ describe('runList', () => { expect(block).toContain('nextToken: next-please'); }); + it('text output selects/reorders columns and suppresses the header', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + pageSize: 25, + columns: 'name,id', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const block = out.join('\n'); + expect(block).toMatch(/^Checkout\s+project_b3c91efa$/); + expect(block).not.toContain('NAME'); + expect(block).not.toContain('CREATED'); + }); + + it('text output rejects unknown columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runList( + { + profile: 'default', + output: 'text', + debug: false, + pageSize: 25, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json output ignores text-only column flags', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'json', + debug: false, + pageSize: 25, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(JSON.parse(out.join('\n')).items[0].id).toBe('project_b3c91efa'); + }); + it('text output reads "No projects." when items is empty and nextToken is null', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); diff --git a/src/commands/project.ts b/src/commands/project.ts index 0a843e5..6a0b332 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -12,6 +12,7 @@ import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { assertNotLocal } from '../lib/target-url.js'; +import { renderTextTable, resolveTextColumns, type TextTableColumn } from '../lib/text-table.js'; import { assertIdempotencyKey } from '../lib/validate.js'; import { fetchSinglePage, @@ -44,6 +45,8 @@ interface ListOptions extends CommonOptions { pageSize?: number; startingToken?: string; maxItems?: number; + columns?: string; + noHeader?: boolean; } export async function runList( @@ -57,6 +60,9 @@ export async function runList( startingToken: opts.startingToken, maxItems: opts.maxItems, }); + if (opts.output === 'text') { + resolveTextColumns(opts.columns, PROJECT_LIST_COLUMNS); + } const client = makeClient(opts, deps); // When the user explicitly passed a page-size flag and did NOT ask @@ -84,7 +90,7 @@ export async function runList( out.print(page, data => { const p = data as Page; - return renderProjectListText(p); + return renderProjectListText(p, { columns: opts.columns, noHeader: opts.noHeader }); }); return page; } @@ -663,6 +669,8 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--page-size ', 'service page-size hint (1-100, default 25)') .option('--starting-token ', 'opaque cursor from a previous list response') .option('--max-items ', 'stop after this many items across auto-paged pages') + .option('--columns ', 'select/reorder text table columns (comma-separated keys)') + .option('--no-header', 'suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (cmdOpts: ListFlagOpts, command: Command) => { // Don't parse numeric flags via Commander — its parser throws a @@ -676,6 +684,8 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { pageSize: parseFlag(cmdOpts.pageSize, 'page-size'), startingToken: cmdOpts.startingToken, maxItems: parseFlag(cmdOpts.maxItems, 'max-items'), + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -895,6 +905,8 @@ interface ListFlagOpts { pageSize?: string; startingToken?: string; maxItems?: string; + columns?: string; + header?: boolean; } interface CreateFlagOpts { @@ -1001,45 +1013,37 @@ function makeOutput(mode: OutputMode, deps: ProjectDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } -function renderProjectListText(page: Page): string { +const PROJECT_LIST_COLUMNS: ReadonlyArray> = [ + { + header: 'ID', + width: rows => Math.max(2, ...rows.map(project => project.id.length)), + render: project => project.id, + }, + { + header: 'NAME', + width: rows => Math.max(4, ...rows.map(project => project.name.length)), + render: project => project.name, + }, + { header: 'TYPE', width: 8, render: project => project.type }, + { header: 'FROM', width: 6, render: project => project.createdFrom }, + { header: 'CREATED', width: 0, render: project => project.createdAt }, +]; + +function renderProjectListText( + page: Page, + options: { columns?: string; noHeader?: boolean } = {}, +): string { if (page.items.length === 0) { return page.nextToken ? `No projects on this page.\nnextToken: ${page.nextToken}` : 'No projects.'; } - // Compact, AWS-CLI-grade columnar output. Column widths are computed - // per-call so a single absurdly long project name doesn't push the - // whole table off-screen. - const idWidth = Math.max(2, ...page.items.map(p => p.id.length)); - const nameWidth = Math.max(4, ...page.items.map(p => p.name.length)); - const typeWidth = 8; - const fromWidth = 6; - - const header = - pad('ID', idWidth) + - ' ' + - pad('NAME', nameWidth) + - ' ' + - pad('TYPE', typeWidth) + - ' ' + - pad('FROM', fromWidth) + - ' ' + - 'CREATED'; - - const rows = page.items.map( - p => - pad(p.id, idWidth) + - ' ' + - pad(p.name, nameWidth) + - ' ' + - pad(p.type, typeWidth) + - ' ' + - pad(p.createdFrom, fromWidth) + - ' ' + - p.createdAt, - ); - - const lines = [header, ...rows]; + const lines = [ + renderTextTable(page.items, PROJECT_LIST_COLUMNS, { + columns: options.columns, + noHeader: options.noHeader, + }), + ]; if (page.nextToken) lines.push('', `nextToken: ${page.nextToken}`); return lines.join('\n'); } @@ -1055,11 +1059,6 @@ function renderProjectText(p: CliProject): string { ].join('\n'); } -function pad(s: string, width: number): string { - if (s.length >= width) return s; - return s + ' '.repeat(width - s.length); -} - function renderUpdateText(r: CliUpdateProjectResponse): string { return [ `id: ${r.id}`, diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index a2d2018..87df3e0 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -225,6 +225,132 @@ describe('runResultHistory — text mode', () => { expect(output).toMatch(/DURATION/); }); + it('selects/reorders columns and suppresses header/separator/detail sub-lines', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_url_001', + targetUrl: 'https://staging.example.com/checkout', + targetUrlSource: 'run', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'status,runid', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output.split('\n')[0]).toMatch(/^passed\s+run_url_001$/); + expect(output).not.toMatch(/^RUN ID/m); + expect(output).not.toMatch(/^-+$/m); + expect(output).not.toContain('targetUrl:'); + }); + + it('no-header suppresses only the history header and separator', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_url_001', + targetUrl: 'https://staging.example.com/checkout', + targetUrlSource: 'run', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).not.toMatch(/^RUN ID/m); + expect(output).not.toMatch(/^-+$/m); + expect(output).toContain('run_url_001'); + expect(output).toContain('targetUrl: https://staging.example.com/checkout'); + }); + + it('rejects unknown history columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json mode ignores text-only history column flags', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: makeHistoryResp() }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const parsed = JSON.parse(lines.join('')) as { runs: Array<{ runId: string }> }; + expect(parsed.runs[0]?.runId).toBe('run_hist_001'); + }); + it('renders run_hist_001 row with status passed and source cli', async () => { const { credentialsPath } = makeCreds(); const lines: string[] = []; diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index bd238d9..b6455a2 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -202,6 +202,7 @@ describe('createTestCommand — surface', () => { it('result exposes --include-analysis (M2.1) + M3.4 piece-5 --history flags', () => { // M2.1 piece 3 adds `--include-analysis` to `test result`. // M3.4 piece 5 adds `--history`, `--source`, `--since`, `--page-size`, `--cursor`. + // Issue #165 adds text-table shaping via `--columns` and `--no-header`. // Pinning the surface so a future flag-consolidation sweep keeps every // option intentional. Back-compat: bare `test result ` (no --history) // still calls runResult and returns the M2 CliLatestResult shape. @@ -215,6 +216,8 @@ describe('createTestCommand — surface', () => { '--since', '--page-size', '--cursor', + '--columns', + '--no-header', ]); }); @@ -577,6 +580,74 @@ describe('runList', () => { expect(block).toContain('mcp'); }); + it('text mode selects/reorders columns and suppresses the header', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [FE_TEST, BE_TEST], nextToken: null }, + })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'status,id', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const lines = out.join('\n').split('\n'); + expect(lines[0]).toMatch(/^failed\s+test_fe$/); + expect(lines[1]).toMatch(/^passed\s+test_be$/); + expect(out.join('\n')).not.toContain('STATUS'); + expect(out.join('\n')).not.toContain('UPDATED'); + }); + + it('text mode rejects unknown columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json mode ignores text-only column flags', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [FE_TEST], nextToken: null }, + })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(JSON.parse(out.join('\n')).items[0].id).toBe('test_fe'); + }); + it('text mode reads "No tests." when items is empty and nextToken is null', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); diff --git a/src/commands/test.ts b/src/commands/test.ts index 4a594dd..7839ca7 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -82,6 +82,13 @@ import type { } from '../lib/runs.types.js'; import { RUN_SOURCES } from '../lib/runs.types.js'; import { assertNotLocal } from '../lib/target-url.js'; +import { + formatTextTableRow, + measureTextColumns, + renderTextTable, + resolveTextColumns, + type TextTableColumn, +} from '../lib/text-table.js'; import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; @@ -493,6 +500,8 @@ interface ListOptions extends CommonOptions { pageSize?: number; startingToken?: string; maxItems?: number; + columns?: string; + noHeader?: boolean; } const TEST_TYPES: ReadonlyArray<'frontend' | 'backend'> = ['frontend', 'backend']; @@ -558,6 +567,9 @@ export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise

renderTestListText(data as Page)); + out.print(page, data => + renderTestListText(data as Page, { columns: opts.columns, noHeader: opts.noHeader }), + ); return page; } @@ -4543,6 +4557,8 @@ interface ResultHistoryOptions extends CommonOptions { pageSize?: number; /** Opaque cursor from a prior page's `nextCursor`. */ cursor?: string; + columns?: string; + noHeader?: boolean; } /** @@ -4571,6 +4587,9 @@ export async function runResultHistory( throw localValidationError('page-size', 'must be between 1 and 100'); } } + if (opts.output === 'text') { + resolveTextColumns(opts.columns, RUN_HISTORY_TABLE_COLUMNS); + } const client = makeClient(opts, deps); const pageSize = opts.pageSize ?? 20; @@ -4583,7 +4602,7 @@ export async function runResultHistory( since: sinceIso, }); - if (opts.output === 'json') { + if (opts.output !== 'text') { out.print({ runs: resp.runs, nextCursor: resp.nextCursor }, data => JSON.stringify(data)); return resp; } @@ -4620,7 +4639,7 @@ export async function runResultHistory( } const lines: string[] = []; - lines.push(renderRunHistoryTable(resp.runs)); + lines.push(renderRunHistoryTable(resp.runs, { columns: opts.columns, noHeader: opts.noHeader })); // Footer: pointer to per-run detail commands. lines.push(''); @@ -4647,13 +4666,18 @@ export async function runResultHistory( return resp; } -const RUN_HISTORY_TABLE_COL_WIDTHS = { - runId: 36, - status: 10, - source: 18, - rerun: 6, - when: 25, -}; +const RUN_HISTORY_TABLE_COLUMNS: ReadonlyArray> = [ + { header: 'RUN ID', width: 36, render: run => run.runId }, + { header: 'STATUS', width: 10, render: run => run.status }, + { header: 'SOURCE', width: 18, render: run => run.source }, + { header: 'RERUN?', width: 6, render: run => (run.isRerun ? 'yes' : 'no') }, + { header: 'WHEN', width: 25, render: run => run.createdAt }, + { + header: 'DURATION', + width: 0, + render: run => formatDurationMs(run.startedAt ?? run.createdAt, run.finishedAt), + }, +]; /** * Max width of the `test steps` DESCRIPTION column in text mode. Long / @@ -4689,60 +4713,41 @@ const HISTORY_TARGET_URL_MAX = 80; * run row (truncated to `HISTORY_TARGET_URL_MAX` chars). The table columns * are left intact to avoid width blow-out on terminals. */ -function renderRunHistoryTable(runs: RunHistoryItem[]): string { - const cols = RUN_HISTORY_TABLE_COL_WIDTHS; - const header = [ - padEnd('RUN ID', cols.runId), - padEnd('STATUS', cols.status), - padEnd('SOURCE', cols.source), - padEnd('RERUN?', cols.rerun), - padEnd('WHEN', cols.when), - 'DURATION', - ].join(' '); - const sep = '-'.repeat(header.length); - - const rows = runs.flatMap(r => { - // FE runs never populate `startedAt` today — the RUNNING heartbeat - // that would set it doesn't fire on the legacy/sync execution path - // (dogfood 2026-06-04), so without a fallback DURATION was always - // "—" for every FE run. Fall back to `createdAt` so the column shows - // wall-clock from trigger to finish; on sync dev the queue gap is - // ~0, and `--output json` still exposes raw startedAt/finishedAt for - // consumers that need to exclude queue time. - const duration = formatDurationMs(r.startedAt ?? r.createdAt, r.finishedAt); - const mainRow = [ - padEnd(r.runId, cols.runId), - padEnd(r.status, cols.status), - padEnd(r.source, cols.source), - padEnd(r.isRerun ? 'yes' : 'no', cols.rerun), - padEnd(r.createdAt, cols.when), - duration, - ].join(' '); +function renderRunHistoryTable( + runs: RunHistoryItem[], + options: { columns?: string; noHeader?: boolean } = {}, +): string { + const selectedColumns = resolveTextColumns(options.columns, RUN_HISTORY_TABLE_COLUMNS); + const widths = measureTextColumns(runs, selectedColumns); + const customColumns = options.columns !== undefined && options.columns.trim() !== ''; + const includeDetailLines = !customColumns; + const header = formatTextTableRow( + selectedColumns.map(column => column.header), + widths, + ); + + const rows = runs.flatMap(run => { + const mainRow = formatTextTableRow( + selectedColumns.map(column => column.render(run)), + widths, + ); - // G1b: surface per-run targetUrl as an indented sub-line. - // Render only when truthy (skip null, undefined, empty) and when the - // source is not 'unresolved' (that would mean "backend couldn't resolve - // a URL" — printing "—" is less informative than omitting the line). const lines: string[] = [mainRow]; - if (r.targetUrl && r.targetUrlSource !== 'unresolved') { + if (includeDetailLines && run.targetUrl && run.targetUrlSource !== 'unresolved') { const url = - r.targetUrl.length > HISTORY_TARGET_URL_MAX - ? `${r.targetUrl.slice(0, HISTORY_TARGET_URL_MAX - 1)}…` - : r.targetUrl; + run.targetUrl.length > HISTORY_TARGET_URL_MAX + ? `${run.targetUrl.slice(0, HISTORY_TARGET_URL_MAX - 1)}…` + : run.targetUrl; lines.push(` targetUrl: ${url}`); - } else if (r.targetUrlSource === 'unresolved') { + } else if (includeDetailLines && run.targetUrlSource === 'unresolved') { lines.push(` targetUrl: —`); } return lines; }); - return [header, sep, ...rows].join('\n'); -} - -function padEnd(s: string, width: number): string { - if (s.length >= width) return s; - return s + ' '.repeat(width - s.length); + if (options.noHeader === true) return rows.join('\n'); + return [header, '-'.repeat(header.length), ...rows].join('\n'); } function formatDurationMs(startedAt: string | null, finishedAt: string | null): string { @@ -8543,6 +8548,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { 'alias for --starting-token; accepted for parity with `test result --history`', ) .option('--max-items ', 'stop after this many items across auto-paged pages') + .option('--columns ', 'select/reorder text table columns (comma-separated keys)') + .option('--no-header', 'suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (cmdOpts: ListFlagOpts, command: Command) => { // Same parser strategy as `project list`: skip Commander's number @@ -8563,6 +8570,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { pageSize: parseNumericFlag(cmdOpts.pageSize, 'page-size'), startingToken: cmdOpts.startingToken ?? cmdOpts.cursor, maxItems: parseNumericFlag(cmdOpts.maxItems, 'max-items'), + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -8891,6 +8900,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option('--page-size ', 'with --history: number of runs per page (1–100, default 20)') .option('--cursor ', 'with --history: opaque cursor from a prior page') + .option('--columns ', 'with --history: select/reorder text table columns') + .option('--no-header', 'with --history: suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testId: string, cmdOpts: ResultFlagOpts, command: Command) => { if (cmdOpts.history) { @@ -8906,6 +8917,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { ? parseNumericFlag(cmdOpts.pageSize, 'page-size') : undefined, cursor: cmdOpts.cursor, + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -9712,6 +9725,8 @@ interface ResultFlagOpts { pageSize?: string; /** Opaque pagination cursor from a prior page's nextCursor. */ cursor?: string; + columns?: string; + header?: boolean; } interface CreateFlagOpts { @@ -9759,6 +9774,8 @@ interface ListFlagOpts { */ cursor?: string; maxItems?: string; + columns?: string; + header?: boolean; } interface StepsFlagOpts { @@ -10106,45 +10123,36 @@ async function streamPresignedBody(url: string, out: Output, deps: TestDeps): Pr } } -function renderTestListText(page: Page): string { +const TEST_LIST_COLUMNS: ReadonlyArray> = [ + { + header: 'ID', + width: rows => Math.max(2, ...rows.map(test => test.id.length)), + render: test => test.id, + }, + { + header: 'NAME', + width: rows => Math.max(4, ...rows.map(test => test.name.length)), + render: test => test.name, + }, + { header: 'TYPE', width: 8, render: test => test.type }, + { header: 'FROM', width: 6, render: test => test.createdFrom }, + { header: 'STATUS', width: 9, render: test => test.status }, + { header: 'UPDATED', width: 0, render: test => test.updatedAt }, +]; + +function renderTestListText( + page: Page, + options: { columns?: string; noHeader?: boolean } = {}, +): string { if (page.items.length === 0) { return page.nextToken ? `No tests on this page.\nnextToken: ${page.nextToken}` : 'No tests.'; } - const idWidth = Math.max(2, ...page.items.map(t => t.id.length)); - const nameWidth = Math.max(4, ...page.items.map(t => t.name.length)); - const typeWidth = 8; - const fromWidth = 6; - const statusWidth = 9; - - const header = - pad('ID', idWidth) + - ' ' + - pad('NAME', nameWidth) + - ' ' + - pad('TYPE', typeWidth) + - ' ' + - pad('FROM', fromWidth) + - ' ' + - pad('STATUS', statusWidth) + - ' ' + - 'UPDATED'; - - const rows = page.items.map( - t => - pad(t.id, idWidth) + - ' ' + - pad(t.name, nameWidth) + - ' ' + - pad(t.type, typeWidth) + - ' ' + - pad(t.createdFrom, fromWidth) + - ' ' + - pad(t.status, statusWidth) + - ' ' + - t.updatedAt, - ); - - const lines = [header, ...rows]; + const lines = [ + renderTextTable(page.items, TEST_LIST_COLUMNS, { + columns: options.columns, + noHeader: options.noHeader, + }), + ]; if (page.nextToken) lines.push('', `nextToken: ${page.nextToken}`); return lines.join('\n'); } diff --git a/src/lib/text-table.ts b/src/lib/text-table.ts new file mode 100644 index 0000000..bd66b5d --- /dev/null +++ b/src/lib/text-table.ts @@ -0,0 +1,94 @@ +import { localValidationError } from './errors.js'; + +export interface TextTableColumn { + header: string; + width: number | ((rows: readonly T[]) => number); + render: (row: T) => string; +} + +export interface TextTableOptions { + columns?: string; + noHeader?: boolean; + separator?: boolean; +} + +export function renderTextTable( + rows: readonly T[], + columns: readonly TextTableColumn[], + options: TextTableOptions = {}, +): string { + const selected = resolveTextColumns(options.columns, columns); + const widths = measureTextColumns(rows, selected); + const body = rows.map(row => + formatTextTableRow( + selected.map(column => column.render(row)), + widths, + ), + ); + + if (options.noHeader === true) return body.join('\n'); + + const header = formatTextTableRow( + selected.map(column => column.header), + widths, + ); + return [header, ...(options.separator === true ? ['-'.repeat(header.length)] : []), ...body].join( + '\n', + ); +} + +export function resolveTextColumns( + raw: string | undefined, + columns: readonly TextTableColumn[], +): readonly TextTableColumn[] { + if (raw === undefined || raw.trim() === '') return columns; + + const byKey = new Map(columns.map(column => [textColumnKey(column.header), column])); + const validKeys = columns.map(column => textColumnKey(column.header)); + const requested = raw.split(',').map(token => token.trim()); + + if (requested.some(token => token.length === 0)) { + throw localValidationError( + 'columns', + `must be a comma-separated list of: ${validKeys.join(', ')}`, + validKeys, + ); + } + + return requested.map(token => { + const key = textColumnKey(token); + const column = byKey.get(key); + if (column === undefined) { + throw localValidationError( + 'columns', + `unknown column "${token}"; must be one of: ${validKeys.join(', ')}`, + validKeys, + ); + } + return column; + }); +} + +export function measureTextColumns( + rows: readonly T[], + columns: readonly TextTableColumn[], +): number[] { + return columns.map(column => + typeof column.width === 'function' ? column.width(rows) : column.width, + ); +} + +export function formatTextTableRow(values: readonly string[], widths: readonly number[]): string { + return values + .map((value, index) => (index === values.length - 1 ? value : pad(value, widths[index] ?? 0))) + .join(' '); +} + +function textColumnKey(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9]/g, ''); +} + +function pad(value: string, width: number): string { + if (value.length >= width) return value; + return value + ' '.repeat(width - value.length); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 441e700..9d364c1 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -195,6 +195,9 @@ Options: --page-size service page-size hint (1-100, default 25) --starting-token opaque cursor from a previous list response --max-items stop after this many items across auto-paged pages + --columns select/reorder text table columns (comma-separated + keys) + --no-header suppress the text table header row -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -506,6 +509,9 @@ Options: --cursor alias for --starting-token; accepted for parity with \`test result --history\` --max-items stop after this many items across auto-paged pages + --columns select/reorder text table columns (comma-separated + keys) + --no-header suppress the text table header row -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -613,6 +619,8 @@ Options: --page-size with --history: number of runs per page (1–100, default 20) --cursor with --history: opaque cursor from a prior page + --columns with --history: select/reorder text table columns + --no-header with --history: suppress the text table header row -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): From 326fa97aae538a3069395f7aa06e51a0e5d6df3c Mon Sep 17 00:00:00 2001 From: Lex Date: Sat, 18 Jul 2026 07:02:21 +0700 Subject: [PATCH 075/117] test(test): guard status filter validation order (#222) Co-authored-by: Lexiie <28455136+Lexiie@users.noreply.github.com> --- src/commands/test.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index b6455a2..c8fe5a5 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -470,6 +470,14 @@ describe('runList', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); }); + it('rejects --status=junk before reading credentials', async () => { + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['list', '--project', 'project_alice', '--status', 'junk'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + it('rejects --page-size=0 locally with VALIDATION_ERROR (no network call)', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => { From 5fe0b0012c7a16209caaeee2bb0771ea0da4d39c Mon Sep 17 00:00:00 2001 From: Shalahuddin Al-Ayyubi <147135570+0xshalah@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:03:49 +0700 Subject: [PATCH 076/117] fix(config): normalize empty/whitespace TESTSPRITE_PROFILE to unset (#233) --- src/lib/config.test.ts | 17 +++++++++++++++++ src/lib/config.ts | 10 +++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/lib/config.test.ts b/src/lib/config.test.ts index cf56eaf..2a58fc8 100644 --- a/src/lib/config.test.ts +++ b/src/lib/config.test.ts @@ -55,6 +55,23 @@ describe('loadConfig', () => { ).toBe('option-profile'); }); + it('treats empty TESTSPRITE_PROFILE as unset (falls back to default)', () => { + expect(loadConfig({ env: { TESTSPRITE_PROFILE: '' }, credentialsPath }).profile).toBe( + 'default', + ); + }); + + it('treats whitespace-only TESTSPRITE_PROFILE as unset (falls back to default)', () => { + const config = loadConfig({ env: { TESTSPRITE_PROFILE: ' ' }, credentialsPath }); + expect(config.profile).toBe('default'); + }); + + it('reads credentials from the default profile when TESTSPRITE_PROFILE is blank', () => { + writeProfile('default', { apiKey: 'sk-default' }, { path: credentialsPath }); + const config = loadConfig({ env: { TESTSPRITE_PROFILE: ' ' }, credentialsPath }); + expect(config.apiKey).toBe('sk-default'); + }); + it('option.endpointUrl overrides everything', () => { writeProfile('default', { apiUrl: 'https://file' }, { path: credentialsPath }); const config = loadConfig({ diff --git a/src/lib/config.ts b/src/lib/config.ts index 7f8065f..a69f2e4 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -39,13 +39,17 @@ export function defaultConfigPath(): string { */ export function loadConfig(options: LoadConfigOptions = {}): Config { const env = options.env ?? process.env; - const profile = options.profile ?? env.TESTSPRITE_PROFILE ?? DEFAULT_PROFILE; + const profile = options.profile ?? normalizeEnvVar(env.TESTSPRITE_PROFILE) ?? DEFAULT_PROFILE; const credentialsPath = options.credentialsPath ?? defaultCredentialsPath(); const fileEntry = readProfile(profile, { path: credentialsPath }); // Empty / whitespace-only env vars are treated as unset so they do not - // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` in a shell - // profile). Matches the normalization in auth configure and init/setup. + // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` or + // `export TESTSPRITE_PROFILE=` in a shell profile). For the profile this + // also avoids a confusing VALIDATION_ERROR: an empty name fails the INI + // section-name guard, so without normalization a blank env var would break + // every command instead of falling back to the default profile. Matches the + // normalization in auth configure and init/setup. const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); const envApiKey = normalizeEnvVar(env.TESTSPRITE_API_KEY); From 2708a4069fba72822f94906d05001b4d19e66d9c Mon Sep 17 00:00:00 2001 From: Tasfia Chowdhury Date: Sat, 18 Jul 2026 06:04:01 +0600 Subject: [PATCH 077/117] fix(init): skip API key prompt when credentials already exist (#234) Add --skip-if-configured to testsprite setup (and the deprecated init alias). When the active profile already has a saved API key and no explicit key source (--api-key or --from-env) is given, the interactive prompt is skipped and the existing key is reused. Motivation: re-running setup to refresh the agent skill -- a common pattern in dotfiles, onboarding scripts, and CI bootstraps -- currently always re-prompts for the key, even when credentials were already configured. The flag makes setup idempotent for the credential step. Behaviour: - runConfigure short-circuits with status:already_configured when skipIfConfigured is true and existingProfile.apiKey is present. - runInit relaxes the non-interactive guard and the --output json guard when skipWillApply is true, so the flag works in CI (isTTY=false) and JSON mode without requiring a separate --api-key. - --api-key always overwrites, regardless of --skip-if-configured. - --from-env always overwrites, regardless of --skip-if-configured. - --dry-run is unaffected (no network, no writes; preview only). New tests (8 cases across auth.test.ts and init.test.ts): - skips prompt and returns early when credentials exist (text + JSON) - proceeds to prompt when no credentials exist - allows isTTY=false CI runs when skip applies - --api-key overwrites despite skip flag - --from-env overwrites despite skip flag Closes #206 --- src/commands/auth.test.ts | 87 +++++++++++++++++ src/commands/auth.ts | 27 +++++- src/commands/init.test.ts | 97 +++++++++++++++++++ src/commands/init.ts | 40 +++++++- test/__snapshots__/help.snapshot.test.ts.snap | 28 +++--- 5 files changed, 261 insertions(+), 18 deletions(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 6f8ce94..0ce0bfb 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -1281,3 +1281,90 @@ describe('createAuthCommand surface', () => { expect(err.exitCode).toBe(3); }); }); + +// --------------------------------------------------------------------------- +// runConfigure -- skipIfConfigured +// --------------------------------------------------------------------------- + +describe('runConfigure -- skipIfConfigured', () => { + it('skips the prompt and returns early when credentials already exist', async () => { + const { capture, deps } = makeCapture(); + // Write a saved key first. + writeProfile('default', { apiKey: 'sk-existing' }, { path: credentialsPath }); + const prompt = { secret: vi.fn(async () => 'sk-new') }; + const fetchImpl = vi.fn(); + + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false, skipIfConfigured: true }, + { + ...deps, + credentialsPath, + prompt, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ); + + // Prompt must never have fired. + expect(prompt.secret).not.toHaveBeenCalled(); + // No network call -- we never validated or wrote a key. + expect(fetchImpl).not.toHaveBeenCalled(); + // The saved key must be untouched. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-existing'); + // Output indicates already_configured. + expect(capture.stdout.join('\n')).toContain('already configured'); + }); + + it('emits already_configured status in JSON mode', async () => { + const { capture, deps } = makeCapture(); + writeProfile('default', { apiKey: 'sk-saved' }, { path: credentialsPath }); + const fetchImpl = vi.fn(); + + await runConfigure( + { profile: 'default', output: 'json', debug: false, fromEnv: false, skipIfConfigured: true }, + { ...deps, credentialsPath, fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'] }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + const parsed = JSON.parse(capture.stdout.join('')); + expect(parsed).toMatchObject({ profile: 'default', status: 'already_configured' }); + }); + + it('proceeds normally when no credentials exist and skipIfConfigured is true', async () => { + const { deps } = makeCapture(); + // No pre-existing profile -- skip has no effect, should fall through to prompt. + const prompt = { secret: vi.fn(async () => 'sk-new') }; + + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false, skipIfConfigured: true }, + { ...deps, credentialsPath, prompt, fetchImpl: meOkFetch }, + ); + + expect(prompt.secret).toHaveBeenCalledTimes(1); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-new'); + }); + + it('ignores skipIfConfigured when --from-env is set', async () => { + const { deps } = makeCapture(); + // Pre-existing key -- but fromEnv should override and write a new one. + writeProfile('default', { apiKey: 'sk-old' }, { path: credentialsPath }); + + await runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + skipIfConfigured: true, + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-from-env' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + + // The env key must overwrite the saved key. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-from-env'); + }); +}); diff --git a/src/commands/auth.ts b/src/commands/auth.ts index e594066..7e60c05 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -71,6 +71,17 @@ type CommonOptions = FactoryCommonOptions; interface ConfigureOptions extends CommonOptions { fromEnv: boolean; + /** + * When true and the active profile already has a saved API key, skip the + * interactive key prompt and proceed directly to the skill-install step. + * A CI-safe flag: lets `setup` run idempotently without prompting on + * machines that already have credentials (e.g. re-running setup to + * refresh the agent skill without re-entering the key). + * + * Ignored when an explicit `--api-key` or `--from-env` key source is + * provided -- those paths always overwrite, regardless of existing state. + */ + skipIfConfigured?: boolean; } const DEFAULT_API_URL = 'https://api.testsprite.com'; @@ -131,9 +142,23 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): apiKey = env.TESTSPRITE_API_KEY?.trim(); if (!apiKey) throw validationError('TESTSPRITE_API_KEY', FROM_ENV_MISSING_KEY); } else { + // --skip-if-configured: when a non-empty API key is already saved for + // this profile, skip the interactive prompt and return early. The + // key is NOT re-validated via GET /me on this path -- the caller + // (runInit) only reaches this when no explicit key source was given, + // and the subsequent whoami call in runInit will surface an expired + // key to the user anyway. + if (opts.skipIfConfigured && existingProfile?.apiKey) { + out.print({ profile: opts.profile, apiUrl, status: 'already_configured' }, data => { + const d = data as { profile: string; apiUrl: string }; + return `Profile "${d.profile}" already configured. Endpoint: ${d.apiUrl}`; + }); + return; + } + const promptApi = deps.prompt ?? { secret: (q: string) => promptSecret(q) }; prelude(`Configuring profile "${opts.profile}".\n`); - // Only the API key is prompted — the endpoint defaults to prod (see above). + // Only the API key is prompted -- the endpoint defaults to prod (see above). apiKey = (await promptApi.secret('TestSprite API key: ')).trim(); if (!apiKey) throw new CLIError('No API key provided.', 5); } diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 0c1bd8e..92e2095 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -9,6 +9,7 @@ import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ApiError, CLIError } from '../lib/errors.js'; import { resetDryRunBannerForTesting } from '../lib/client-factory.js'; +import { readProfile, writeProfile } from '../lib/credentials.js'; import type { MeResponse } from './auth.js'; import type { AgentFs } from './agent.js'; import type { InitDeps } from './init.js'; @@ -1045,3 +1046,99 @@ describe('runInit — telemetry attribution (X-CLI-Command)', () => { expect(initTagged).toHaveLength(1); }); }); + +// --------------------------------------------------------------------------- +// runInit -- skipIfConfigured +// --------------------------------------------------------------------------- + +describe('runInit -- skipIfConfigured', () => { + it('skips the API key prompt and reuses saved credentials when the profile exists', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + // Write a saved key before running setup. + writeProfile('default', { apiKey: 'sk-saved' }, { path: credentialsPath }); + // Provide a mock fetch that accepts /me so runWhoami (identity banner) succeeds. + const fetchMock = makeOkFetch(); + const prompt = { secret: vi.fn(async () => 'sk-should-never-be-asked') }; + + await runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'json' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + prompt, + }); + + // The prompt must never have fired. + expect(prompt.secret).not.toHaveBeenCalled(); + // The saved key must be untouched. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-saved'); + // The summary must still be emitted. + const parsed = JSON.parse(captured.stdout.join('')) as { status: string }; + expect(parsed.status).toBe('initialized'); + }); + + it('proceeds to prompt when skipIfConfigured is true but no credentials exist', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + // No pre-existing credentials -- skip has no effect. + const fetchMock = makeOkFetch(); + const prompt = { secret: vi.fn(async () => 'sk-fresh') }; + + await runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'text' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: true, + prompt, + }); + + // With no saved key, the prompt should fire. + expect(prompt.secret).toHaveBeenCalledTimes(1); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-fresh'); + expect(captured.stdout.join('')).toContain('initialized'); + }); + + it('allows non-interactive (isTTY=false) when skipIfConfigured is true and credentials exist', async () => { + const { deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + writeProfile('default', { apiKey: 'sk-ci' }, { path: credentialsPath }); + const fetchMock = makeOkFetch(); + + // Must not throw exit 5 for "non-interactive mode, no key source". + await expect( + runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'json' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + }), + ).resolves.toBeUndefined(); + + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-ci'); + }); + + it('--api-key takes precedence over skipIfConfigured and overwrites the saved key', async () => { + const { deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + writeProfile('default', { apiKey: 'sk-old' }, { path: credentialsPath }); + const fetchMock = makeOkFetch(); + + await runInit( + makeBaseOpts({ apiKey: 'sk-new', skipIfConfigured: true, noAgent: true, output: 'text' }), + { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + }, + ); + + // Explicit --api-key must overwrite regardless of skipIfConfigured. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-new'); + }); +}); diff --git a/src/commands/init.ts b/src/commands/init.ts index 6357e3b..659945f 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -90,6 +90,12 @@ interface InitOptions extends CommonOptions { force: boolean; dir?: string; yes: boolean; + /** + * When true and the active profile already has a saved API key, skip the + * interactive key prompt. Forwarded verbatim to runConfigure. Has no + * effect when --api-key or --from-env is also given. + */ + skipIfConfigured?: boolean; /** Set by the command action when both --agent and --no-agent appear in rawArgs. */ rawArgConflict?: boolean; } @@ -196,9 +202,16 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise, --from-env (reads TESTSPRITE_API_KEY), or run interactively.', @@ -208,7 +221,7 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise or --from-env.', @@ -223,7 +236,13 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise', 'Project root for the skill install (default: current directory)') - .option('-y, --yes', 'Non-interactive: accept all defaults, never prompt'); + .option('-y, --yes', 'Non-interactive: accept all defaults, never prompt') + .option( + '--skip-if-configured', + 'Skip the API key prompt when credentials already exist for this profile (CI-safe idempotent re-run)', + ); } /** Build {@link InitOptions} from raw Commander opts + globals. */ @@ -543,6 +572,7 @@ function buildSetupOptions( force: Boolean(cmdOpts.force), dir: cmdOpts.dir, yes: Boolean(cmdOpts.yes), + skipIfConfigured: Boolean(cmdOpts.skipIfConfigured), rawArgConflict, }; } diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 9d364c1..4724064 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -112,18 +112,22 @@ exports[`--help snapshots > init 1`] = ` (deprecated) alias for \`setup\` Options: - --api-key API key to configure (skips the interactive prompt) - --from-env Read TESTSPRITE_API_KEY from the environment instead of - prompting (default: false) - --agent Coding-agent target to install: claude, antigravity, - cursor, cline, kiro, windsurf, copilot, codex (default: - claude) (default: "claude") - --no-agent Skip the agent skill install (configure credentials only) - --force Overwrite an existing skill file (a .bak backup is kept) - --dir Project root for the skill install (default: current - directory) - -y, --yes Non-interactive: accept all defaults, never prompt - -h, --help display help for command + --api-key API key to configure (skips the interactive prompt) + --from-env Read TESTSPRITE_API_KEY from the environment instead of + prompting (default: false) + --agent Coding-agent target to install: claude, antigravity, + cursor, cline, kiro, windsurf, copilot, codex (default: + claude) (default: "claude") + --no-agent Skip the agent skill install (configure credentials + only) + --force Overwrite an existing skill file (a .bak backup is + kept) + --dir Project root for the skill install (default: current + directory) + -y, --yes Non-interactive: accept all defaults, never prompt + --skip-if-configured Skip the API key prompt when credentials already exist + for this profile (CI-safe idempotent re-run) + -h, --help display help for command " `; From 7975f6ff510b47ad0d5006bb91fc99a839798f16 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 04:52:53 +0200 Subject: [PATCH 078/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index a6f68d3..ba0d84f 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4819,6 +4819,48 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { }); }); +// --------------------------------------------------------------------------- +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty +// --------------------------------------------------------------------------- +describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } + return errorBody('NOT_FOUND'); + }); + + const stdoutLines: string[] = []; + const err = await runTestRerun( + { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false }, + fetchImpl, + (line) => stdoutLines.push(line) + ); + + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + }); +}); + // --------------------------------------------------------------------------- // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- From 51af9393d0671acc6824cb5dbecafd4115060fc8 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 04:58:55 +0200 Subject: [PATCH 079/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index ba0d84f..e275b48 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4824,6 +4824,7 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { // --------------------------------------------------------------------------- describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const creds = makeCreds(); const batchResp: BatchRerunResponse = { accepted: [ { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, @@ -4864,7 +4865,6 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol // --------------------------------------------------------------------------- // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- - describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); From 7dd2576a1f07bc9fe0370b000b7aca3792cd6c9d Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:05:11 +0300 Subject: [PATCH 080/117] Add newline at end of test.rerun.spec.ts Fix missing newline at end of file. From 15ee2dccc53280b269eda2ceefebd1539865858d Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:09:33 +0300 Subject: [PATCH 081/117] Add newline at end of test.rerun.spec.ts Fix missing newline at end of file in test.rerun.spec.ts From 770cd4788de000559689f39a4ca8c3ce8e6c1911 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:16:06 +0300 Subject: [PATCH 082/117] Refactor runTestRerun call with new parameters --- src/commands/test.rerun.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index e275b48..672d270 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4848,9 +4848,8 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const stdoutLines: string[] = []; const err = await runTestRerun( { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false }, - fetchImpl, - (line) => stdoutLines.push(line) - ); + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (line) => stdoutLines.push(line), stderr: () => undefined } + ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { From f7120c25f34a54fda7f487f43e3533d98b0ca3d3 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:22:08 +0300 Subject: [PATCH 083/117] Add newline at end of test.rerun.spec.ts Fix missing newline at end of file in test.rerun.spec.ts From 0b34b1adc1d984d04cfb37bcf1d8b08b83b585c1 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:32:14 +0300 Subject: [PATCH 084/117] Add newline at end of test.rerun.spec.ts Fix missing newline at end of file in test.rerun.spec.ts From 5e2bf88cbed0b214736ac7da152bc25298f849a0 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:26:01 +0200 Subject: [PATCH 085/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 672d270..4428f8f 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4847,7 +4847,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const stdoutLines: string[] = []; const err = await runTestRerun( - { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false }, + { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false maxConcurrency: 1, profile: 'default', output: 'json'}, { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (line) => stdoutLines.push(line), stderr: () => undefined } ).catch(e => e); From 98f9393df3d16c0998ca43bddc15349319008627 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:35:05 +0300 Subject: [PATCH 086/117] Refactor test parameters for runTestRerun function --- src/commands/test.rerun.spec.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 4428f8f..8958606 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4847,8 +4847,26 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const stdoutLines: string[] = []; const err = await runTestRerun( - { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false maxConcurrency: 1, profile: 'default', output: 'json'}, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (line) => stdoutLines.push(line), stderr: () => undefined } + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 60, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 1, + profile: 'default', + output: 'json', + debug: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); From c4bef009e228fe810e832d2e922749568463de09 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:44:11 +0300 Subject: [PATCH 087/117] Fix formatting and comments in test.rerun.spec.ts Fixed formatting issues by adding a missing comma and clarifying comments. --- src/commands/test.rerun.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 8958606..3ff9859 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4855,7 +4855,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 1, + maxConcurrency: 1, // Fixed: Added comma and missing fields profile: 'default', output: 'json', debug: false, @@ -4864,7 +4864,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), + stdout: line => stdoutLines.push(line), stderr: () => undefined, }, ).catch(e => e); From 0a3bfe352c8ffc47c603cd4b8fda7ba312e0d0b2 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:50:13 +0200 Subject: [PATCH 088/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 3ff9859..0fdde9f 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4855,18 +4855,18 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 1, // Fixed: Added comma and missing fields + maxConcurrency: 1, // تأكدت من وجود الفواصل هنا profile: 'default', output: 'json', - debug: false, + debug: false }, { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: line => stdoutLines.push(line), - stderr: () => undefined, - }, + stderr: () => undefined + } ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); From 9a6a48dec23600fb964d9f7ef69d73a61cf9e2f0 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:54:04 +0200 Subject: [PATCH 089/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 0fdde9f..2e97da3 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4855,7 +4855,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 1, // تأكدت من وجود الفواصل هنا + maxConcurrency: 1, profile: 'default', output: 'json', debug: false From 19376059eba9f303a15ff094b4b33e6de07774fc Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:58:11 +0200 Subject: [PATCH 090/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 55 +++++++++------------------------ 1 file changed, 14 insertions(+), 41 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 2e97da3..fd39488 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4855,7 +4855,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 1, + maxConcurrency: 1, profile: 'default', output: 'json', debug: false @@ -4870,12 +4870,9 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')) as { - accepted: Array<{ testId: string; runId: string; status: string }>; - }; + const parsed = JSON.parse(stdoutLines.join('\n')); expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + expect(parsed.accepted.map((r: any) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); }); }); @@ -4885,39 +4882,20 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); - const rerunResp = makeFeRerunResp(); + const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - let fetchCallCount = 0; - const fetchImpl: typeof globalThis.fetch = async (input, _init) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL - ? input.toString() - : (input as { url: string }).url; - fetchCallCount++; - if (url.includes('/tests/test_fe_01/runs/rerun')) { - return new Response(JSON.stringify(rerunResp), { - status: 202, - headers: { 'content-type': 'application/json' }, - }); + const fetchImpl: any = async (input: any) => { + const url = typeof input === 'string' ? input : input.url; + if (url.includes('/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { status: 202 }); } if (url.includes('/runs/')) { - const runningRun: RunResponse = { - ...makeTerminalRun(rerunResp.runId, 'passed'), - status: 'running', - finishedAt: null, - }; - return new Response(JSON.stringify(runningRun), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); + return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); } return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); }; const stdoutLines: string[] = []; - const err = await runTestRerun( { testIds: ['test_fe_01'], @@ -4930,25 +4908,20 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t maxConcurrency: 10, output: 'json', profile: 'default', - dryRun: false, - debug: false, - verbose: false, + debug: false }, { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: line => stdoutLines.push(line), - stderr: () => undefined, - }, + stderr: () => undefined + } ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); - expect(stdoutLines.length).toBeGreaterThan(0); - const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + const parsed = JSON.parse(stdoutLines.join('\n')); expect(parsed.runId).toBe(rerunResp.runId); expect(parsed.status).toBe('running'); - - void fetchCallCount; }); -}); +}); \ No newline at end of file From f6aea0e2d948f15a6c3d30c6d90f9ad10e0e6aee Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:23:29 +0200 Subject: [PATCH 091/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 137 +++----------------------------- 1 file changed, 12 insertions(+), 125 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index e08b850..396e84b 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4819,7 +4819,6 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); - // --------------------------------------------------------------------------- // Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- @@ -4829,23 +4828,17 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const batchResp: BatchRerunResponse = { accepted: [ { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' } ], deferred: [], conflicts: [], - closure: { byProject: [] }, + closure: { byProject: [] } }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) { - return { status: 202, body: batchResp }; - } - if (url.includes('/runs/')) { - throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); - } + const fetchImpl = makeFetch((url) => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch'); return errorBody('NOT_FOUND'); }); - const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4861,19 +4854,9 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol output: 'json', debug: false }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: line => stdoutLines.push(line), - stderr: () => undefined - } - ).catch(e => e); - + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as any, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')); - expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map((r: any) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); }); }); @@ -4884,18 +4867,12 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - const fetchImpl: any = async (input: any) => { const url = typeof input === 'string' ? input : input.url; - if (url.includes('/runs/rerun')) { - return new Response(JSON.stringify(rerunResp), { status: 202 }); - } - if (url.includes('/runs/')) { - return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); - } + if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); + if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); }; - const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4903,59 +4880,6 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t all: false, wait: true, timeoutSeconds: 0, -// DEV-331 piece 1 — graceful detach during batch rerun --wait (SIG-6) -// --------------------------------------------------------------------------- - -describe('R-BAT: batch rerun --wait — InterruptError partial lists all dispatched runIds (DEV-331)', () => { - it('interrupt mid fan-out → stdout partial covers every accepted runId, honest stderr, exit 130', async () => { - const creds = makeCreds(); - const shutdown = new ShutdownController(); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - // Batch trigger resolves; every run poll hangs until the composed signal aborts. - const fetchImpl: FetchImpl = (async (input: unknown, init: RequestInit = {}) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL - ? input.toString() - : (input as { url: string }).url; - if (url.includes('/tests/batch/rerun')) { - return new Response(JSON.stringify(batchResp), { - status: 202, - headers: { 'content-type': 'application/json' }, - }); - } - return new Promise((_resolve, reject) => { - const signal = init.signal; - const rejectWithReason = (): void => { - const reason: unknown = signal?.reason; - reject(reason instanceof Error ? reason : new Error('aborted')); - }; - if (signal?.aborted) { - rejectWithReason(); - return; - } - signal?.addEventListener('abort', rejectWithReason, { once: true }); - }); - }) as FetchImpl; - - const stdoutLines: string[] = []; - const stderrLines: string[] = []; - const pending = runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 600, autoHeal: false, autoHealExplicit: false, skipDependencies: false, @@ -4963,49 +4887,12 @@ describe('R-BAT: batch rerun --wait — InterruptError partial lists all dispatc output: 'json', profile: 'default', debug: false - dryRun: false, - debug: false, - verbose: false, }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: line => stdoutLines.push(line), - stderr: () => undefined - } - ).catch(e => e); - + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as any, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')); expect(parsed.runId).toBe(rerunResp.runId); expect(parsed.status).toBe('running'); }); -}); - fetchImpl, - stdout: line => stdoutLines.push(line), - stderr: line => stderrLines.push(line), - shutdown, - }, - ); - setTimeout(() => shutdown.interrupt('SIGINT'), 10); - - const err = await pending.catch(e => e); - expect(err).toBeInstanceOf(InterruptError); - expect((err as InterruptError).exitCode).toBe(130); - - // SIG-6: the partial lists ALL dispatched runIds, marked running. - const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { - accepted: Array<{ runId: string; status: string }>; - }; - const byRunId = new Map(stdoutJson.accepted.map(r => [r.runId, r.status])); - expect(byRunId.get('run_b1')).toBe('running'); - expect(byRunId.get('run_b2')).toBe('running'); - - const stderrBlock = stderrLines.join('\n'); - expect(stderrBlock).toContain('Interrupted (SIGINT)'); - expect(stderrBlock).toContain('billing'); - expect(stderrBlock).toContain('run_b1'); - expect(stderrBlock).toContain('run_b2'); - }); -}); +}); \ No newline at end of file From ad0dd10d31706135185314654ecf242085df0959 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:32:16 +0200 Subject: [PATCH 092/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 396e84b..2c1b7ba 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4819,6 +4819,7 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); + // --------------------------------------------------------------------------- // Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- @@ -4854,7 +4855,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol output: 'json', debug: false }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as any, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); }); @@ -4867,8 +4868,8 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - const fetchImpl: any = async (input: any) => { - const url = typeof input === 'string' ? input : input.url; + const fetchImpl: FetchImpl = async (input) => { + const url = typeof input === 'string' ? input : (input as Request).url; if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); @@ -4888,10 +4889,10 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t profile: 'default', debug: false }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as any, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; expect(parsed.runId).toBe(rerunResp.runId); expect(parsed.status).toBe('running'); }); From 817781274d798d0bdf3a15b822367c897eb4c42d Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:35:15 +0200 Subject: [PATCH 093/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 58 ++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 2c1b7ba..f569b61 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4829,17 +4829,19 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const batchResp: BatchRerunResponse = { accepted: [ { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' } + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, ], deferred: [], conflicts: [], - closure: { byProject: [] } + closure: { byProject: [] }, }; + const fetchImpl = makeFetch((url) => { if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch'); + if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); return errorBody('NOT_FOUND'); }); + const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4853,11 +4855,24 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol maxConcurrency: 1, profile: 'default', output: 'json', - debug: false + debug: false, }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + } ).catch((e) => e); + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted?: Array<{ testId?: string; runId: string; status?: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); }); }); @@ -4865,15 +4880,27 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { - it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + it('exit 7 AND stdout contains {runId, status:\"running\"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - const fetchImpl: FetchImpl = async (input) => { - const url = typeof input === 'string' ? input : (input as Request).url; - if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); - if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); + + const fetchImpl: FetchImpl = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { status: 202 }); + } + if (url.includes('/runs/')) { + return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); + } return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); }; + const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4887,10 +4914,17 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t maxConcurrency: 10, output: 'json', profile: 'default', - debug: false + debug: false, }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + } ).catch((e) => e); + expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; expect(parsed.runId).toBe(rerunResp.runId); From bdec093a27edfe1eb75e8f7c1be0186ee11af3ca Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:40:33 +0200 Subject: [PATCH 094/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 58 +++++++-------------------------- 1 file changed, 12 insertions(+), 46 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index f569b61..2c1b7ba 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4829,19 +4829,17 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const batchResp: BatchRerunResponse = { accepted: [ { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' } ], deferred: [], conflicts: [], - closure: { byProject: [] }, + closure: { byProject: [] } }; - const fetchImpl = makeFetch((url) => { if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch'); return errorBody('NOT_FOUND'); }); - const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4855,24 +4853,11 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol maxConcurrency: 1, profile: 'default', output: 'json', - debug: false, + debug: false }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), - stderr: () => undefined, - } + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } ).catch((e) => e); - expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')) as { - accepted?: Array<{ testId?: string; runId: string; status?: string }>; - }; - expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); }); }); @@ -4880,27 +4865,15 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { - it('exit 7 AND stdout contains {runId, status:\"running\"} when --timeout polling deadline is exceeded', async () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - - const fetchImpl: FetchImpl = async (input, _init) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL - ? input.toString() - : (input as { url: string }).url; - - if (url.includes('/runs/rerun')) { - return new Response(JSON.stringify(rerunResp), { status: 202 }); - } - if (url.includes('/runs/')) { - return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); - } + const fetchImpl: FetchImpl = async (input) => { + const url = typeof input === 'string' ? input : (input as Request).url; + if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); + if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); }; - const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4914,17 +4887,10 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t maxConcurrency: 10, output: 'json', profile: 'default', - debug: false, + debug: false }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), - stderr: () => undefined, - } + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } ).catch((e) => e); - expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; expect(parsed.runId).toBe(rerunResp.runId); From 6b080ac62e829b848c36b6eeb12618aae20b064e Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:44:37 +0200 Subject: [PATCH 095/117] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 199 +++++++++++++++++++++----------- 1 file changed, 132 insertions(+), 67 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 2c1b7ba..966d16b 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4823,77 +4823,142 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { // --------------------------------------------------------------------------- // Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- -describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { - it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { - const creds = makeCreds(); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' } - ], - deferred: [], - conflicts: [], - closure: { byProject: [] } - }; - const fetchImpl = makeFetch((url) => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch'); - return errorBody('NOT_FOUND'); - }); - const stdoutLines: string[] = []; - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 60, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 1, - profile: 'default', - output: 'json', - debug: false +describe( + '[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', + () => { + it( + 'stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', + async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch((url) => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } + return errorBody('NOT_FOUND'); + }); + + const stdoutLines: string[] = []; + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 60, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 1, + profile: 'default', + output: 'json', + debug: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch((e) => e); + + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } - ).catch((e) => e); - expect(err).toMatchObject({ exitCode: 7 }); - }); -}); + ); + }, +); // --------------------------------------------------------------------------- // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { - it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { - const creds = makeCreds(); - const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - const fetchImpl: FetchImpl = async (input) => { - const url = typeof input === 'string' ? input : (input as Request).url; - if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); - if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); - return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); - }; - const stdoutLines: string[] = []; - const err = await runTestRerun( - { - testIds: ['test_fe_01'], - all: false, - wait: true, - timeoutSeconds: 0, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 10, - output: 'json', - profile: 'default', - debug: false - }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } - ).catch((e) => e); - expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; - expect(parsed.runId).toBe(rerunResp.runId); - expect(parsed.status).toBe('running'); - }); + it( + 'exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', + async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + let fetchCallCount = 0; + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + fetchCallCount++; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + const runningRun: RunResponse = { + ...makeTerminalRun(rerunResp.runId, 'passed'), + status: 'running', + finishedAt: null, + }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 0, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch((e) => e); + + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(parsed.runId).toBe(rerunResp.runId); + expect(parsed.status).toBe('running'); + + void fetchCallCount; + }, + ); }); \ No newline at end of file From 44501e068a1c0bb655f47e316147bed1fd203ac9 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 08:51:58 +0300 Subject: [PATCH 096/117] Refactor batch rerun tests for RequestTimeoutError --- src/commands/test.rerun.spec.ts | 241 +++++++++++++++----------------- 1 file changed, 116 insertions(+), 125 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 966d16b..ea05c9a 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4823,142 +4823,133 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { // --------------------------------------------------------------------------- // Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- -describe( - '[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', - () => { - it( - 'stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', - async () => { - const creds = makeCreds(); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch((url) => { - if (url.includes('/tests/batch/rerun')) { - return { status: 202, body: batchResp }; - } - if (url.includes('/runs/')) { - throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); - } - return errorBody('NOT_FOUND'); - }); +describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; - const stdoutLines: string[] = []; - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 60, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 1, - profile: 'default', - output: 'json', - debug: false, - }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), - stderr: () => undefined, - }, - ).catch((e) => e); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } + return errorBody('NOT_FOUND'); + }); - expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')) as { - accepted: Array<{ testId: string; runId: string; status: string }>; - }; - expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); + const stdoutLines: string[] = []; + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 60, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 1, + profile: 'default', + output: 'json', + debug: false, }, - ); - }, -); + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch(e => e); + + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + }); +}); // --------------------------------------------------------------------------- // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { - it( - 'exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', - async () => { - const creds = makeCreds(); - const rerunResp = makeFeRerunResp(); - - let fetchCallCount = 0; - const fetchImpl: typeof globalThis.fetch = async (input, _init) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + let fetchCallCount = 0; + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL ? input.toString() : (input as { url: string }).url; - fetchCallCount++; - if (url.includes('/tests/test_fe_01/runs/rerun')) { - return new Response(JSON.stringify(rerunResp), { - status: 202, - headers: { 'content-type': 'application/json' }, - }); - } - if (url.includes('/runs/')) { - const runningRun: RunResponse = { - ...makeTerminalRun(rerunResp.runId, 'passed'), - status: 'running', - finishedAt: null, - }; - return new Response(JSON.stringify(runningRun), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - } - return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); - }; + fetchCallCount++; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + const runningRun: RunResponse = { + ...makeTerminalRun(rerunResp.runId, 'passed'), + status: 'running', + finishedAt: null, + }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; - const stdoutLines: string[] = []; + const stdoutLines: string[] = []; - const err = await runTestRerun( - { - testIds: ['test_fe_01'], - all: false, - wait: true, - timeoutSeconds: 0, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 10, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), - stderr: () => undefined, - }, - ).catch((e) => e); + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 0, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch(e => e); - expect(err).toMatchObject({ exitCode: 7 }); - expect(stdoutLines.length).toBeGreaterThan(0); - const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; - expect(parsed.runId).toBe(rerunResp.runId); - expect(parsed.status).toBe('running'); + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(parsed.runId).toBe(rerunResp.runId); + expect(parsed.status).toBe('running'); - void fetchCallCount; - }, - ); -}); \ No newline at end of file + void fetchCallCount; + }); +}); From 7774c29dff80bfc0edc94fa89176500bdcfe1dc1 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 09:02:38 +0300 Subject: [PATCH 097/117] Refactor test.rerun.spec.ts imports Removed InterruptError and ShutdownController imports. --- src/commands/test.rerun.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index ea05c9a..a35cfd0 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -9,8 +9,8 @@ import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; -import { ShutdownController } from '../lib/interrupt.js'; +import { ApiError, RequestTimeoutError } from '../lib/errors.js'; + import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; import type { FetchImpl } from '../lib/http.js'; import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; From 73441e43e4b54c56bec0ca7671bd925a8f27826b Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 08:27:38 +0200 Subject: [PATCH 098/117] Refactor fetch implementation and error handling --- src/commands/test.rerun.spec.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index a35cfd0..81283d9 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -10,7 +10,6 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { ApiError, RequestTimeoutError } from '../lib/errors.js'; - import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; import type { FetchImpl } from '../lib/http.js'; import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; @@ -4836,7 +4835,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol closure: { byProject: [] }, }; - const fetchImpl = makeFetch(url => { + const fetchImpl = makeFetch((url) => { if (url.includes('/tests/batch/rerun')) { return { status: 202, body: batchResp }; } @@ -4868,15 +4867,15 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol stdout: (line) => stdoutLines.push(line), stderr: () => undefined, }, - ).catch(e => e); + ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { accepted: Array<{ testId: string; runId: string; status: string }>; }; expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); }); }); @@ -4939,10 +4938,10 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: line => stdoutLines.push(line), + stdout: (line) => stdoutLines.push(line), stderr: () => undefined, }, - ).catch(e => e); + ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); expect(stdoutLines.length).toBeGreaterThan(0); From 0081f3ff4f0a4820b7a3a4f9324f7fd15db7f509 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 08:41:26 +0200 Subject: [PATCH 099/117] Fix formatting --- src/commands/test.rerun.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 81283d9..e018e88 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -2889,7 +2889,7 @@ describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', ...creds, sleep: instantSleep, fetchImpl, - stdout: () => {}, + stdout: () => { }, stderr: line => stderrLines.push(line), }, ); @@ -2945,7 +2945,7 @@ describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', ...creds, sleep: instantSleep, fetchImpl, - stdout: () => {}, + stdout: () => { }, stderr: line => stderrLines.push(line), }, ); @@ -3177,7 +3177,7 @@ describe('runTestRerun --all --skip-terminal (dogfood L1796)', () => { ...creds, sleep: instantSleep, fetchImpl: makeFilterFetch(dispatched), - stderr: () => {}, + stderr: () => { }, }, ); From c286766a346e65fa7cd327f0392901607ed954fc Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:48:05 +0000 Subject: [PATCH 100/117] Fix Prettier formatting strictly --- package-lock.json | 4 ++-- src/commands/test.rerun.spec.ts | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4461016..f9afe03 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index e018e88..c4e865a 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -2889,7 +2889,7 @@ describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', ...creds, sleep: instantSleep, fetchImpl, - stdout: () => { }, + stdout: () => {}, stderr: line => stderrLines.push(line), }, ); @@ -2945,7 +2945,7 @@ describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', ...creds, sleep: instantSleep, fetchImpl, - stdout: () => { }, + stdout: () => {}, stderr: line => stderrLines.push(line), }, ); @@ -3177,7 +3177,7 @@ describe('runTestRerun --all --skip-terminal (dogfood L1796)', () => { ...creds, sleep: instantSleep, fetchImpl: makeFilterFetch(dispatched), - stderr: () => { }, + stderr: () => {}, }, ); @@ -4835,7 +4835,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol closure: { byProject: [] }, }; - const fetchImpl = makeFetch((url) => { + const fetchImpl = makeFetch(url => { if (url.includes('/tests/batch/rerun')) { return { status: 202, body: batchResp }; } @@ -4864,18 +4864,18 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), + stdout: line => stdoutLines.push(line), stderr: () => undefined, }, - ).catch((e) => e); + ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { accepted: Array<{ testId: string; runId: string; status: string }>; }; expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); @@ -4938,10 +4938,10 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), + stdout: line => stdoutLines.push(line), stderr: () => undefined, }, - ).catch((e) => e); + ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); expect(stdoutLines.length).toBeGreaterThan(0); From 774db845970a9e4826ff9823c8e6a343a0a0f049 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 09:26:20 +0200 Subject: [PATCH 101/117] Re-run failed jobs --- src/commands/test.rerun.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index c4e865a..9f1f4ef 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4952,3 +4952,4 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t void fetchCallCount; }); }); + \ No newline at end of file From 943b76198ab486b07567b7531ac0daff8d984933 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 09:27:37 +0200 Subject: [PATCH 102/117] Re-run failed jobs --- src/commands/test.rerun.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 9f1f4ef..c4e865a 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4952,4 +4952,3 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t void fetchCallCount; }); }); - \ No newline at end of file From 8f463b333c7c2c8aa9866f454a7987345b843ee0 Mon Sep 17 00:00:00 2001 From: Kshitij Bhardwaj Date: Tue, 21 Jul 2026 21:14:57 +0530 Subject: [PATCH 103/117] fix(test): create-batch --run without --wait exits 0 on a fully successful dispatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The no-wait fan-out returns each result with the trigger response's status, which is `queued` by design (non-terminal — the field is documented as "Terminal status if --wait; queued if no --wait"). But the exit-code logic only recognized terminal statuses: const allPassed = batchRunResults.every(r => r.status === 'passed'); Every no-wait result is `queued`, so `allPassed` was always false and the command fell through to exit 1 with a misleading "N of N run(s) did not pass" — even when every trigger was dispatched successfully. This broke the documented exit-code contract (the DOCUMENTATION.md create-batch example uses `--run --max-concurrency 4 --output json`, no `--wait`), failed CI on fully successful dispatches, and was internally inconsistent with single `test run ` without `--wait`, which exits 0 on a successful `queued` dispatch. In the no-wait case "success" means every trigger dispatched without error (statuses are non-terminal by definition). The exit-code block now branches on `opts.wait`: const failing = opts.wait ? batchRunResults.filter(r => r.status !== 'passed') : batchRunResults.filter(r => r.error !== undefined); if (failing.length === 0) return; // exit 0 Trigger errors in no-wait mode keep today's aggregation (uniform specific code when shared, else exit 1), and the "did not pass" message becomes "N of N trigger(s) failed" when `--wait` is absent. The stderr text summary is likewise corrected for no-wait mode — it now reports "N/N triggered" instead of misreading a successful dispatch as "0/N passed". Adds three specs in test.create-batch-run.spec.ts: all-queued (json) → resolves without error and never polls; all-queued (text) → "N/N triggered" summary, no "passed"/"did not pass" wording; partial trigger failure → still exits non-zero with the full results envelope. The two success specs fail against the pre-fix exit-code block and pass with it. Existing no-wait specs only exercised error scenarios and never asserted the success exit code, which is how this slipped through. Fixes #161 --- DOCUMENTATION.md | 2 +- src/commands/test.create-batch-run.spec.ts | 219 +++++++++++++++++++++ src/commands/test.ts | 44 +++-- 3 files changed, 250 insertions(+), 15 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 700671b..cb14d9a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -271,7 +271,7 @@ testsprite test create --plan-from ./checkout.plan.json --dry-run --output json #### `testsprite test create-batch` -Bulk-create frontend tests from a JSONL plan-steps file (or a directory of plan files with `--plan-from-dir`). Optional `--run --max-concurrency ` fans out triggers. +Bulk-create frontend tests from a JSONL plan-steps file (or a directory of plan files with `--plan-from-dir`). Optional `--run --max-concurrency ` fans out triggers. Without `--wait`, each run is dispatched (`status: "queued"`) and the command exits 0 when every trigger is accepted — mirroring single `test run` without `--wait`; a trigger error still exits non-zero. With `--wait`, it polls every run to terminal and exits non-zero if any run does not pass. ```bash testsprite test create-batch --plans ./plans.jsonl --run --max-concurrency 4 --output json diff --git a/src/commands/test.create-batch-run.spec.ts b/src/commands/test.create-batch-run.spec.ts index 5719543..db2c3c1 100644 --- a/src/commands/test.create-batch-run.spec.ts +++ b/src/commands/test.create-batch-run.spec.ts @@ -333,6 +333,225 @@ describe('runCreateBatch --run --wait: mixed outcomes', () => { }); }); +// --------------------------------------------------------------------------- +// No --wait: exit-code contract (issue #161) +// +// Without --wait, every trigger response is non-terminal ('queued') by design. +// A fully successful dispatch must exit 0 — success means every trigger was +// dispatched without error, mirroring single `test run` (no --wait). A trigger +// error must still exit non-zero. Existing no-wait specs only exercised error +// scenarios and never asserted the success exit code, which is how the +// "always exits 1 even when every trigger succeeds" bug slipped through. +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run (no --wait): exit-code contract', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('all triggers succeed (queued) → resolves without error, exit 0; results all queued, no error (json)', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_q1', 'test_q2', 'test_q3']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC, FE_SPEC]); + + let pollCount = 0; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + return { body: makeTriggerResponse(testId, `run_${testId}`) }; + } + // No --wait must NOT poll — count any GET /runs as a violation. + if (/\/runs\/run_test_[a-z0-9]+/.exec(url)) { + pollCount++; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const stdout: string[] = []; + const stderrLines: string[] = []; + + // Must NOT throw — a fully successful no-wait dispatch exits 0. + // (If runBatchRun threw a CLIError, this await would reject and fail the test.) + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + expect(pollCount).toBe(0); // no polling without --wait + + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string; error?: unknown }>; + }; + expect(printed.results).toHaveLength(3); + expect(printed.results.every(r => r.status === 'queued')).toBe(true); + expect(printed.results.every(r => r.error === undefined)).toBe(true); + }); + + it('all triggers succeed (queued) → text summary reports "3/3 triggered", exit 0 (no "0/N passed")', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_t1', 'test_t2', 'test_t3']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC, FE_SPEC]); + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + return { body: makeTriggerResponse(testId, `run_${testId}`) }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const stdout: string[] = []; + const stderrLines: string[] = []; + + // Must NOT throw — a fully successful no-wait dispatch exits 0. + await runCreateBatch( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + const summary = stderrLines.find(l => l.startsWith('batch-run summary:')); + expect(summary).toBeDefined(); + expect(summary).toContain('3/3 triggered'); + // The pre-fix bug printed a pass/fail summary ("0/3 passed") for a fully + // successful no-wait dispatch — assert that misleading wording is gone. + expect(summary).not.toContain('passed'); + expect(stderrLines.some(l => l.includes('did not pass'))).toBe(false); + }); + + it('partial trigger failure (2 queued, 1 errors) → still exits non-zero; all 3 results in envelope', async () => { + const { credentialsPath } = makeCreds(); + const testIds = ['test_p1', 'test_p2', 'test_p3']; + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC, FE_SPEC]); + + // test_p3's trigger returns 404 NOT_FOUND — a non-retryable error (exit 4) + // that surfaces immediately as an error result. The other two dispatch fine. + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) { + return { body: makeBatchCreateResponse(testIds) }; + } + const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); + if (triggerMatch?.[1]) { + const testId = triggerMatch[1]; + if (testId === 'test_p3') { + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'test not found', + nextAction: '', + requestId: 'req_p3', + }, + }, + }; + } + return { body: makeTriggerResponse(testId, `run_${testId}`) }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const stdout: string[] = []; + const stderrLines: string[] = []; + + const err = await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(e => e); + + // A dispatch with any trigger error must exit non-zero. + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).exitCode).not.toBe(0); + + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string; error?: { code: string } }>; + }; + expect(printed.results).toHaveLength(3); + const queued = printed.results.filter(r => r.status === 'queued'); + const errored = printed.results.filter(r => r.error !== undefined); + expect(queued).toHaveLength(2); + expect(errored).toHaveLength(1); + expect(errored[0]?.testId).toBe('test_p3'); + expect(errored[0]?.error?.code).toBe('NOT_FOUND'); + }); +}); + // --------------------------------------------------------------------------- // --max-concurrency: verify only N in-flight at any time // --------------------------------------------------------------------------- diff --git a/src/commands/test.ts b/src/commands/test.ts index c6e6c31..69ce51a 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -2888,6 +2888,17 @@ async function runBatchRun( }) : batchRunResults; out.print({ results: enrichedResults }); + } else if (!opts.wait) { + // Text mode, no --wait: statuses are non-terminal by design ('queued'), + // so a pass/fail summary would misread a successful dispatch as + // "0/N passed". Report what actually happened: triggers. + const total = batchRunResults.length; + const erroredCount = batchRunResults.filter(r => r.error !== undefined).length; + const parts = [`${total - erroredCount}/${total} triggered`]; + if (erroredCount > 0) { + parts.push(`${erroredCount} trigger error${erroredCount !== 1 ? 's' : ''}`); + } + stderrFn(`batch-run summary: ${parts.join(', ')}`); } else { // Text mode: print summary line. const passed = batchRunResults.filter(r => r.status === 'passed').length; @@ -2906,20 +2917,23 @@ async function runBatchRun( stderrFn(`batch-run summary: ${parts.join(', ')}`); } - // Determine exit code. - const allPassed = batchRunResults.every(r => r.status === 'passed'); - if (allPassed) return; // exit 0 + // Determine exit code. With --wait, success means every run reached the + // terminal status 'passed'. Without --wait, statuses are non-terminal by + // design ('queued' per CliBatchRunResult), so success means every trigger + // dispatched without error — mirroring single `test run` (no --wait), + // which exits 0 on a successful queued dispatch. + const failing = opts.wait + ? batchRunResults.filter(r => r.status !== 'passed') + : batchRunResults.filter(r => r.error !== undefined); + if (failing.length === 0) return; // exit 0 - // Check for a uniform non-pass exit code across all non-passed results. - const errorExitCodes = batchRunResults - .filter(r => r.error !== undefined) - .map(r => r.error!.exitCode); - const nonPassedStatuses = batchRunResults.filter(r => r.status !== 'passed'); + // Check for a uniform non-pass exit code across all failing results. + const errorExitCodes = failing.filter(r => r.error !== undefined).map(r => r.error!.exitCode); // Exit 7 only when EVERY run timed out — a mix of pass + timeout is "mixed - // outcomes" (exit 1), not "all timed out". `nonPassedStatuses.every(...)` + // outcomes" (exit 1), not "all timed out". `failing.every(...)` alone // would incorrectly fire exit 7 when 1 of N passed and the rest timed out. const allTimeout = - batchRunResults.length > 0 && + failing.length === batchRunResults.length && batchRunResults.every(r => r.status === 'timeout' || r.error?.exitCode === 7); if (allTimeout) { throw new CLIError( @@ -2927,8 +2941,8 @@ async function runBatchRun( 7, ); } - // If all non-passed results share the same specific exit code (6 or 11), use it. - if (errorExitCodes.length > 0 && errorExitCodes.length === nonPassedStatuses.length) { + // If all failing results share the same specific exit code (6 or 11), use it. + if (errorExitCodes.length > 0 && errorExitCodes.length === failing.length) { const uniformCode = errorExitCodes[0]; if ( uniformCode !== undefined && @@ -2937,14 +2951,16 @@ async function runBatchRun( uniformCode !== 7 ) { throw new CLIError( - `Batch run finished: ${nonPassedStatuses.length} run(s) failed with exit code ${uniformCode}.`, + `Batch run finished: ${failing.length} run(s) failed with exit code ${uniformCode}.`, uniformCode, ); } } // Default: mixed outcomes or generic failure → exit 1. throw new CLIError( - `Batch run finished: ${batchRunResults.filter(r => r.status !== 'passed').length} of ${batchRunResults.length} run(s) did not pass.`, + opts.wait + ? `Batch run finished: ${failing.length} of ${batchRunResults.length} run(s) did not pass.` + : `Batch run trigger finished: ${failing.length} of ${batchRunResults.length} trigger(s) failed.`, 1, ); } From d821dac7924f644ef8086eb1e0820bb0fbc0ee44 Mon Sep 17 00:00:00 2001 From: nopp Date: Fri, 24 Jul 2026 04:46:39 +0700 Subject: [PATCH 104/117] fix(auth): lock credential profile mutations (#272) * fix(auth): lock credential profile mutations * fix(auth): verify credential lock ownership --- src/lib/credentials.test.ts | 14 +++ src/lib/credentials.ts | 174 +++++++++++++++++++++++++++++++++--- 2 files changed, 175 insertions(+), 13 deletions(-) diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index 194c1ff..447d9d3 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -157,6 +157,19 @@ describe('writeProfile', () => { expect(file.dev).toEqual({ apiKey: 'sk-dev', apiUrl: 'https://dev' }); }); + it('reclaims stale mutation locks and removes the lock after writing', () => { + const lockPath = `${credentialsPath}.lock`; + writeFileSync( + lockPath, + `${JSON.stringify({ pid: process.pid, createdAt: Date.now() - 60_000, token: 'stale' })}\n`, + ); + + writeProfile('default', { apiKey: 'sk-new' }, { path: credentialsPath }); + + expect(readProfile('default', { path: credentialsPath })).toEqual({ apiKey: 'sk-new' }); + expect(existsSync(lockPath)).toBe(false); + }); + it('does not leak the api key into the on-disk file format aside from the value itself', () => { writeProfile('default', { apiKey: 'sk-secret-12345' }, { path: credentialsPath }); const onDisk = readFileSync(credentialsPath, 'utf-8'); @@ -168,6 +181,7 @@ describe('writeProfile', () => { describe('deleteProfile', () => { it('returns false when the profile is missing', () => { expect(deleteProfile('nope', { path: credentialsPath })).toBe(false); + expect(existsSync(credentialsPath)).toBe(false); }); it('removes the named profile and leaves others intact', () => { diff --git a/src/lib/credentials.ts b/src/lib/credentials.ts index 6c20b1f..c67944b 100644 --- a/src/lib/credentials.ts +++ b/src/lib/credentials.ts @@ -5,9 +5,11 @@ import { readFileSync, renameSync, statSync, + unlinkSync, writeFileSync, } from 'node:fs'; import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { localValidationError } from './errors.js'; @@ -62,6 +64,17 @@ export interface CredentialsOptions { path?: string; } +interface CredentialsLockInfo { + pid?: number; + createdAt?: number; + token?: string; +} + +interface CredentialsLock { + assertHeld: () => void; + release: () => void; +} + interface RestrictiveModeOptions { platform?: NodeJS.Platform; env?: NodeJS.ProcessEnv; @@ -83,6 +96,10 @@ const FIELD_TO_FILE_KEY: Record = { apiUrl: 'api_url', }; +const CREDENTIALS_LOCK_RETRY_MS = 25; +const CREDENTIALS_LOCK_WAIT_MS = 5_000; +const CREDENTIALS_LOCK_STALE_MS = 30_000; + export function parseCredentials(content: string): CredentialsFile { const result: CredentialsFile = {}; let currentEntry: ProfileEntry | null = null; @@ -165,23 +182,24 @@ export function writeProfile( ): void { assertValidProfileName(profile); const path = resolvePath(options); - const file = readCredentialsFile(options); - file[profile] = { ...file[profile], ...entry }; - writeCredentialsAtomic(path, file); + mutateCredentialsFile(path, file => { + file[profile] = { ...file[profile], ...entry }; + return file; + }); } export function deleteProfile(profile: string, options: CredentialsOptions = {}): boolean { assertValidProfileName(profile); const path = resolvePath(options); - const file = readCredentialsFile(options); - if (!(profile in file)) return false; - delete file[profile]; - if (Object.keys(file).length === 0) { - writeCredentialsAtomic(path, {}); - } else { - writeCredentialsAtomic(path, file); - } - return true; + if (!existsSync(path)) return false; + let removed = false; + mutateCredentialsFile(path, file => { + if (!(profile in file)) return undefined; + removed = true; + delete file[profile]; + return file; + }); + return removed; } /** @@ -244,10 +262,140 @@ function resolvePath(options: CredentialsOptions): string { return options.path ?? defaultCredentialsPath(); } -function writeCredentialsAtomic(path: string, file: CredentialsFile): void { +function mutateCredentialsFile( + path: string, + mutate: (file: CredentialsFile) => CredentialsFile | undefined, +): void { mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const lock = acquireCredentialsLock(path); + try { + const file = readCredentialsFile({ path }); + const nextFile = mutate(file); + if (nextFile === undefined) return; + lock.assertHeld(); + writeCredentialsAtomic(path, nextFile); + } finally { + lock.release(); + } +} + +function writeCredentialsAtomic(path: string, file: CredentialsFile): void { const tmp = `${path}.tmp.${process.pid}`; writeFileSync(tmp, serializeCredentials(file), { mode: 0o600, encoding: 'utf8' }); renameSync(tmp, path); ensureRestrictiveMode(path); } + +function acquireCredentialsLock(path: string): CredentialsLock { + const lockPath = `${path}.lock`; + const deadline = Date.now() + CREDENTIALS_LOCK_WAIT_MS; + const token = `${process.pid}:${Date.now()}:${randomUUID()}`; + const lockInfo: Required = { + pid: process.pid, + createdAt: Date.now(), + token, + }; + + while (true) { + try { + writeFileSync(lockPath, `${JSON.stringify(lockInfo)}\n`, { + encoding: 'utf8', + flag: 'wx', + mode: 0o600, + }); + return { + assertHeld: () => assertCredentialsLockHeld(lockPath, token), + release: () => releaseCredentialsLock(lockPath, token), + }; + } catch (error) { + if (!isErrnoException(error) || error.code !== 'EEXIST') { + throw error; + } + + reclaimStaleCredentialsLock(lockPath); + if (Date.now() >= deadline) { + throw localValidationError( + 'credentialsLock', + 'timed out waiting for another credential update to finish; retry the command', + undefined, + 'field', + ); + } + sleepSync(CREDENTIALS_LOCK_RETRY_MS); + } + } +} + +function reclaimStaleCredentialsLock(lockPath: string): void { + let lockInfo: CredentialsLockInfo | undefined; + try { + lockInfo = JSON.parse(readFileSync(lockPath, 'utf-8')) as CredentialsLockInfo; + } catch { + lockInfo = undefined; + } + + const createdAt = typeof lockInfo?.createdAt === 'number' ? lockInfo.createdAt : undefined; + const ageMs = createdAt === undefined ? Number.POSITIVE_INFINITY : Date.now() - createdAt; + const pid = typeof lockInfo?.pid === 'number' ? lockInfo.pid : undefined; + if (ageMs <= CREDENTIALS_LOCK_STALE_MS && (pid === undefined || isProcessAlive(pid))) { + return; + } + + try { + unlinkSync(lockPath); + } catch (error) { + if (!isErrnoException(error) || error.code !== 'ENOENT') { + throw error; + } + } +} + +function assertCredentialsLockHeld(lockPath: string, token: string): void { + const lockInfo = readCredentialsLockInfo(lockPath); + if (lockInfo?.token === token) return; + throw localValidationError( + 'credentialsLock', + 'lost ownership of the credential update lock; retry the command', + undefined, + 'field', + ); +} + +function releaseCredentialsLock(lockPath: string, token: string): void { + const lockInfo = readCredentialsLockInfo(lockPath); + if (lockInfo?.token !== token) return; + try { + unlinkSync(lockPath); + } catch (error) { + if (!isErrnoException(error) || error.code !== 'ENOENT') { + throw error; + } + } +} + +function readCredentialsLockInfo(lockPath: string): CredentialsLockInfo | undefined { + try { + return JSON.parse(readFileSync(lockPath, 'utf-8')) as CredentialsLockInfo; + } catch { + return undefined; + } +} + +function isProcessAlive(pid: number): boolean { + if (pid === process.pid) return true; + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (isErrnoException(error) && error.code === 'ESRCH') return false; + return true; + } +} + +function isErrnoException(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && 'code' in error; +} + +function sleepSync(ms: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} From 61bc16734a8488eb0206e2db302ccb35af25d270 Mon Sep 17 00:00:00 2001 From: Joseph Jang Date: Thu, 23 Jul 2026 14:47:11 -0700 Subject: [PATCH 105/117] Recovered: feat: support TESTSPRITE_PROJECT_ID default (#144 by @naufalfx805-source) (#249) * feat: support project env default * fix: tighten project env fallback --------- Co-authored-by: nopp --- DOCUMENTATION.md | 27 ++--- src/commands/test.run.spec.ts | 49 +++++++++ src/commands/test.test.ts | 103 +++++++++++++++++- src/commands/test.ts | 90 ++++++++------- test/__snapshots__/help.snapshot.test.ts.snap | 18 +-- 5 files changed, 225 insertions(+), 62 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index c373a8a..8704182 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -600,24 +600,25 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ----------------------------------------- | ---------------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | -| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | -| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | -| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support — API traffic is routed through the configured proxy | -| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | -| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | +| Variable | Purpose | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------ | +| `TESTSPRITE_API_KEY` | API key - overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint - overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_PROJECT_ID` | Default project for `test list`, `test create`, and `test run --all` when `--project` is omitted | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`-`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support - API traffic is routed through the configured proxy | +| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | +| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | ### Update notice Interactive runs print a one-line "new version available" notice on stderr when a newer release exists. To learn this, the CLI contacts the public npm registry (`registry.npmjs.org`) at most once per 24 hours; the request carries the -package name only — never your API key, project data, or command line. The +package name only - never your API key, project data, or command line. The check is skipped in CI, when stderr is not a TTY, under `--output json` / `--dry-run`, and entirely when `TESTSPRITE_NO_UPDATE_NOTIFIER` is set. Any failure is silent: the notice can never break or delay a command. This is the @@ -627,7 +628,7 @@ Separately, the backend advertises its **minimum supported CLI version** on every `/api/cli/v1` response. When the running CLI is below that floor, a one-line upgrade advisory is printed to stderr (same opt-outs as the update notice; it never changes the exit status). A backend may also reject a -too-old client outright with HTTP 426 — surfaced as `CLIENT_TOO_OLD`, +too-old client outright with HTTP 426 - surfaced as `CLIENT_TOO_OLD`, exit `14`, non-retriable, with upgrade guidance. ### Scopes diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 51b527f..7eb8eb0 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -2582,9 +2582,58 @@ describe('runTestRunAll — batch fresh run', () => { await expect(test.parseAsync(['run', '--all'], { from: 'user' })).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5, + details: expect.objectContaining({ + reason: expect.stringContaining('TESTSPRITE_PROJECT_ID'), + }), }); }); + it('run --all uses TESTSPRITE_PROJECT_ID when --project is omitted', async () => { + const { createTestCommand } = await import('./test.js'); + const { credentialsPath } = makeCreds(); + type Captured = { url: string; method: string; body: unknown }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + captured.push({ url, method, body: init.body ? JSON.parse(init.body as string) : undefined }); + return { body: BATCH_FRESH_RESP }; + }); + const test = createTestCommand({ + credentialsPath, + env: { TESTSPRITE_PROJECT_ID: 'project_env' } as NodeJS.ProcessEnv, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }); + await test.parseAsync(['run', '--all'], { from: 'user' }); + const post = captured.find(c => c.method === 'POST' && c.url.includes('/tests/batch/run'))!; + expect(post.body).toMatchObject({ projectId: 'project_env', source: 'cli' }); + }); + + it('run --all uses TESTSPRITE_PROJECT_ID when --project is blank', async () => { + const { createTestCommand } = await import('./test.js'); + const { credentialsPath } = makeCreds(); + type Captured = { url: string; method: string; body: unknown }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + captured.push({ url, method, body: init.body ? JSON.parse(init.body as string) : undefined }); + return { body: BATCH_FRESH_RESP }; + }); + const test = createTestCommand({ + credentialsPath, + env: { TESTSPRITE_PROJECT_ID: 'project_env' } as NodeJS.ProcessEnv, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }); + await test.parseAsync(['run', '--all', '--project', ' '], { from: 'user' }); + const post = captured.find(c => c.method === 'POST' && c.url.includes('/tests/batch/run'))!; + expect(post.body).toMatchObject({ projectId: 'project_env', source: 'cli' }); + }); + it('--all --target-url → exit 5 (target-url has no effect on BE-only batch)', async () => { const { createTestCommand } = await import('./test.js'); const test = createTestCommand(); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index c8fe5a5..1922448 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -365,6 +365,64 @@ describe('runList', () => { expect(seen[0]).toContain('createdFrom=portal'); }); + it('uses TESTSPRITE_PROJECT_ID when --project is omitted', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: { items: [FE_TEST], nextToken: null } }; + }); + await runList( + { profile: 'default', output: 'json', debug: false }, + { + credentialsPath, + env: { TESTSPRITE_PROJECT_ID: 'project_env' } as NodeJS.ProcessEnv, + fetchImpl, + stdout: () => undefined, + }, + ); + expect(seen[0]).toContain('projectId=project_env'); + }); + + it('uses TESTSPRITE_PROJECT_ID when --project is blank', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: { items: [FE_TEST], nextToken: null } }; + }); + await runList( + { profile: 'default', output: 'json', debug: false, projectId: ' ' }, + { + credentialsPath, + env: { TESTSPRITE_PROJECT_ID: 'project_env' } as NodeJS.ProcessEnv, + fetchImpl, + stdout: () => undefined, + }, + ); + expect(seen[0]).toContain('projectId=project_env'); + }); + + it('prefers explicit --project over TESTSPRITE_PROJECT_ID', async () => { + const { credentialsPath } = makeCreds(); + const seen: string[] = []; + const fetchImpl = makeFetch(url => { + seen.push(url); + return { body: { items: [FE_TEST], nextToken: null } }; + }); + await runList( + { profile: 'default', output: 'json', debug: false, projectId: 'project_flag' }, + { + credentialsPath, + env: { TESTSPRITE_PROJECT_ID: 'project_env' } as NodeJS.ProcessEnv, + fetchImpl, + stdout: () => undefined, + }, + ); + expect(seen[0]).toContain('projectId=project_flag'); + expect(seen[0]).not.toContain('projectId=project_env'); + }); + it('accepts --created-from cli and passes createdFrom=cli to the wire (dogfood 2026-06-04)', async () => { // End-to-end through parseEnumFlag: backend now stamps createFrom='cli' // on `testsprite test create` rows, so the filter must accept 'cli'. @@ -783,10 +841,15 @@ describe('createTestCommand list — required flag', () => { await test.parseAsync(['list'], { from: 'user' }); expect.unreachable('expected ApiError'); } catch (err) { - const apiErr = err as { code?: string; exitCode?: number; details?: { field?: string } }; + const apiErr = err as { + code?: string; + exitCode?: number; + details?: { field?: string; reason?: string }; + }; expect(apiErr.code).toBe('VALIDATION_ERROR'); expect(apiErr.exitCode).toBe(5); expect(apiErr.details?.field).toBe('project'); + expect(apiErr.details?.reason).toContain('TESTSPRITE_PROJECT_ID'); } }); @@ -4847,7 +4910,7 @@ describe('runCreate', () => { ...SAMPLE_RESPONSE, type: 'backend', warnings: [ - 'This test appears to hardcode an auth credential — read auth from __AUTH_HEADERS__.', + 'This test appears to hardcode an auth credential - read auth from __AUTH_HEADERS__.', ], }, }; @@ -4873,10 +4936,43 @@ describe('runCreate', () => { ); // Warning lands on stderr, prefixed `[warn]`. expect(err.some(l => l.includes('[warn]') && l.includes('__AUTH_HEADERS__'))).toBe(true); - // stdout stays the parseable wire object — no warning noise. + // stdout stays the parseable wire object - no warning noise. expect(out.join('\n')).not.toContain('[warn]'); }); + it('uses TESTSPRITE_PROJECT_ID for create when --project is omitted', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('code body'); + type Captured = { method: string; body: unknown; url: string }; + const captured: Captured[] = []; + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + captured.push({ url, method, body: init.body ? JSON.parse(init.body as string) : undefined }); + if (method === 'GET') return { status: 200, body: { items: [] } }; + return { status: 200, body: SAMPLE_RESPONSE }; + }); + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'n', + codeFile, + }, + { + credentialsPath, + env: { TESTSPRITE_PROJECT_ID: 'project_env' } as NodeJS.ProcessEnv, + fetchImpl, + stdout: () => undefined, + }, + ); + expect(captured.some(c => c.method === 'GET' && c.url.includes('projectId=project_env'))).toBe( + true, + ); + const post = captured.find(c => c.method === 'POST')!; + expect(post.body).toMatchObject({ projectId: 'project_env' }); + }); it('respects a caller-supplied --idempotency-key (for safe retries)', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('code body'); @@ -5030,7 +5126,6 @@ describe('runCreate', () => { profile: 'default', output: 'json', debug: false, - // @ts-expect-error — exercising the runtime gate projectId: undefined, type: 'frontend', name: 'n', diff --git a/src/commands/test.ts b/src/commands/test.ts index 3a57966..ca6801c 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -487,7 +487,7 @@ function interruptDetachMessage(err: InterruptError, runIds: string[]): string { type CommonOptions = FactoryCommonOptions; interface ListOptions extends CommonOptions { - projectId: string; + projectId?: string; type?: 'frontend' | 'backend'; createdFrom?: 'portal' | 'mcp' | 'cli'; /** @@ -552,7 +552,8 @@ export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise

= { - projectId: opts.projectId, + projectId, type: opts.type, createdFrom: opts.createdFrom, status: opts.status, @@ -642,7 +643,7 @@ export type CliCreatePriority = (typeof CLI_CREATE_PRIORITIES)[number]; const MAX_INLINE_CODE_BYTES = 350 * 1024; interface CreateOptions extends CommonOptions { - projectId: string; + projectId?: string; type: 'frontend' | 'backend'; name: string; description?: string; @@ -790,7 +791,8 @@ export async function runCreate( assertChainedRunKeyFits(opts.run, opts.idempotencyKey); // Validate inputs before touching credentials or fs — matches the // M2 read commands' "input gates first, then auth, then I/O" ordering. - requireProjectId(opts.projectId); + const projectId = resolveProjectId(opts.projectId, deps); + requireProjectId(projectId); requireNonEmpty('name', opts.name); // P1-3: client-side length checks matching server limits (name ≤200, // description ≤2000) so the user gets instant, actionable errors instead @@ -871,7 +873,7 @@ export async function runCreate( } const body: Record = { - projectId: opts.projectId, + projectId, type: opts.type, name: opts.name, description: opts.description, @@ -915,7 +917,7 @@ export async function runCreate( // B3: best-effort duplicate-name advisory. Skip under --dry-run. if (!opts.dryRun) { const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - await emitDupNameAdvisoryIfNeeded(client, opts.projectId, opts.name, stderrFn); + await emitDupNameAdvisoryIfNeeded(client, projectId, opts.name, stderrFn); } const response = await client.post('/tests', { @@ -940,7 +942,7 @@ export async function runCreate( // R1: suppress under --dry-run (fake canned test id). const chainDashboardUrl = opts.dryRun ? undefined - : resolvePortalUrl(resolveApiUrl(opts, deps), opts.projectId, response.testId); + : resolvePortalUrl(resolveApiUrl(opts, deps), projectId, response.testId); const createContextWithUrl = chainDashboardUrl !== undefined ? { ...response, dashboardUrl: chainDashboardUrl } : response; await runTestRun( @@ -970,7 +972,7 @@ export async function runCreate( // (e.g. "test_dryrun_create_2026") and a live-looking URL would mislead. const dashboardUrl = opts.dryRun ? undefined - : resolvePortalUrl(resolveApiUrl(opts, deps), opts.projectId, response.testId); + : resolvePortalUrl(resolveApiUrl(opts, deps), projectId, response.testId); if (opts.output === 'json') { out.print(dashboardUrl !== undefined ? { ...response, dashboardUrl } : response, data => renderCreateText(data as CliCreateTestResponse), @@ -6343,8 +6345,8 @@ export async function runTestWait( // --------------------------------------------------------------------------- interface RunTestRunAllOptions extends CommonOptions { - /** projectId to run all tests in. */ - projectId: string; + /** projectId to run all tests in; may be resolved from --project or TESTSPRITE_PROJECT_ID. */ + projectId?: string; /** --filter : only run tests whose name contains this substring (case-insensitive). */ nameFilter?: string; /** --wait: block until terminal or --timeout. */ @@ -6409,7 +6411,8 @@ export async function runTestRunAll( deps: TestDeps = {}, ): Promise { assertIdempotencyKey(opts.idempotencyKey); - requireProjectId(opts.projectId); + const projectId = resolveProjectId(opts.projectId, deps); + requireProjectId(projectId); if ( !Number.isInteger(opts.maxConcurrency) || opts.maxConcurrency < 1 || @@ -6440,7 +6443,7 @@ export async function runTestRunAll( method: 'POST', path: '/api/cli/v1/tests/batch/run', body: { - projectId: opts.projectId, + projectId, testIds: opts.nameFilter ? [''] : undefined, source: 'cli' as const, }, @@ -6469,9 +6472,9 @@ export async function runTestRunAll( const projectDashboardUrl = batchPortalBase === undefined ? undefined - : `${batchPortalBase}/dashboard/tests/${encodeURIComponent(opts.projectId)}`; + : `${batchPortalBase}/dashboard/tests/${encodeURIComponent(projectId)}`; const withBatchDashboardUrl = (item: T): T => { - const dashboardUrl = resolvePortalUrl(batchApiUrl, opts.projectId, item.testId); + const dashboardUrl = resolvePortalUrl(batchApiUrl, projectId, item.testId); return dashboardUrl !== undefined ? { ...item, dashboardUrl } : item; }; @@ -6487,7 +6490,7 @@ export async function runTestRunAll( const allPage = await paginate( async ({ pageSize, cursor }) => client.get>('/tests', { - query: { projectId: opts.projectId, pageSize, cursor }, + query: { projectId, pageSize, cursor }, }), {}, ); @@ -6503,7 +6506,7 @@ export async function runTestRunAll( testIds = filtered.map(t => t.id); if (testIds.length === 0) { stderrFn( - `No tests found in project ${opts.projectId} matching --filter "${opts.nameFilter}" — nothing to run.`, + `No tests found in project ${projectId} matching --filter "${opts.nameFilter}" — nothing to run.`, ); out.print({ accepted: [], @@ -6515,7 +6518,7 @@ export async function runTestRunAll( return undefined; } stderrFn( - `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch run.`, + `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${projectId} for batch run.`, ); } // When no --filter, omit testIds → server runs ALL tests in the project @@ -6523,7 +6526,7 @@ export async function runTestRunAll( const batchResp = await client.triggerBatchRunFresh( { - projectId: opts.projectId, + projectId, ...(testIds !== undefined ? { testIds } : {}), source: 'cli', }, @@ -6667,7 +6670,7 @@ export async function runTestRunAll( try { retryResp = await client.triggerBatchRunFresh( { - projectId: opts.projectId, + projectId, testIds: retryIds, source: 'cli', }, @@ -9098,12 +9101,12 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--all', - 'run all tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', + 'run all tests in the project (wave-ordered fresh run; uses --project or TESTSPRITE_PROJECT_ID). Mutually exclusive with .', false, ) .option( '--project ', - 'project id (required with --all; returned by `testsprite project list`)', + 'project id (with --all, overrides TESTSPRITE_PROJECT_ID; returned by `testsprite project list`)', ) .option( '--filter ', @@ -9125,9 +9128,11 @@ export function createTestCommand(deps: TestDeps = {}): Command { .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + - ' testsprite test run --all --project run all project tests in wave order\n' + - ' testsprite test run --all --project --filter name-glob subset\n' + - ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + + ' testsprite test run --all --project run all project tests in wave order\n' + + ' TESTSPRITE_PROJECT_ID= testsprite test run --all use env default project\n' + + ' testsprite test run --all --filter name-glob subset (uses --project/env)\n' + + ' testsprite test run --all --wait --report junit --report-file ./results.xml\n' + + ' project id precedence: --project wins over TESTSPRITE_PROJECT_ID\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + '(see `testsprite test create --help` for details).\n' + '\nFrontend tests: the current unified engine runs FE tests too (they are billed\n' + @@ -9148,7 +9153,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (testIdArg === undefined && !isAll) { throw localValidationError( 'test-id', - 'provide a , or use --all --project to run all tests in a project', + 'provide a , or use --all with --project or TESTSPRITE_PROJECT_ID', ); } // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). @@ -9157,7 +9162,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (cmdOpts.filter !== undefined && cmdOpts.filter !== '' && !isAll) { throw localValidationError( 'filter', - '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all --project .', + '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all with --project or TESTSPRITE_PROJECT_ID.', ); } const report = parseJUnitReportFormat(cmdOpts.report); @@ -9171,18 +9176,17 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (isAll) { // --all path: wave-ordered fresh batch run. - if (!cmdOpts.project) { - throw localValidationError( - 'project', - '--all requires a project id — pass --project ', - ); - } + const projectId = resolveProjectId(cmdOpts.project, deps); + requireProjectId( + projectId, + '--all requires a project id - pass --project or set TESTSPRITE_PROJECT_ID', + ); // --target-url has no effect on the --all batch path: a BE test's base // URL is baked into its code, and the unified engine resolves each // project's configured environment server-side (per-run URL overrides // are not applied to batch FE runs either). Silently dropping it could - // run the suite against an unintended environment in the caller's mind - // — reject loudly instead. + // run the suite against an unintended environment in the caller's mind, + // so reject loudly. if (cmdOpts.targetUrl !== undefined && cmdOpts.targetUrl !== '') { throw localValidationError( 'target-url', @@ -9192,7 +9196,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { await runTestRunAll( { ...resolveCommonOptions(command), - projectId: cmdOpts.project, + projectId, nameFilter: cmdOpts.filter, wait: cmdOpts.wait === true, timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), @@ -9801,9 +9805,19 @@ interface StepsFlagOpts { runId?: string; } -function requireProjectId(projectId: string): void { +function resolveProjectId(projectId: string | undefined, deps: TestDeps): string | undefined { + const explicit = projectId?.trim(); + if (explicit && explicit.length > 0) return explicit; + const envValue = (deps.env ?? process.env).TESTSPRITE_PROJECT_ID; + const trimmed = envValue?.trim(); + return trimmed && trimmed.length > 0 ? trimmed : undefined; +} +function requireProjectId( + projectId: string | undefined, + message = 'is required; pass --project or set TESTSPRITE_PROJECT_ID', +): asserts projectId is string { if (typeof projectId !== 'string' || projectId.length === 0) { - throw localValidationError('project', 'is required'); + throw localValidationError('project', message); } } diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 4724064..55a7252 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -664,10 +664,12 @@ Options: --idempotency-key opaque key for safe retries (1–256 chars). Printed to stderr at --debug if auto-generated. --all run all tests in the project (wave-ordered fresh - run; requires --project). Mutually exclusive with - . (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) + run; uses --project or TESTSPRITE_PROJECT_ID). + Mutually exclusive with . (default: + false) + --project project id (with --all, overrides + TESTSPRITE_PROJECT_ID; returned by \`testsprite + project list\`) --filter with --all: only run tests whose name contains this substring (case-insensitive) --max-concurrency with --all --wait, max in-flight polls at once @@ -680,9 +682,11 @@ Options: -h, --help display help for command Dependency-aware fresh run (M4): - testsprite test run --all --project run all project tests in wave order - testsprite test run --all --project --filter name-glob subset - testsprite test run --all --project --wait --report junit --report-file ./results.xml + testsprite test run --all --project run all project tests in wave order + TESTSPRITE_PROJECT_ID= testsprite test run --all use env default project + testsprite test run --all --filter name-glob subset (uses --project/env) + testsprite test run --all --wait --report junit --report-file ./results.xml + project id precedence: --project wins over TESTSPRITE_PROJECT_ID BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). From d78d0d4513b389553745259d28290d22bd8586cc Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:47:33 -0400 Subject: [PATCH 106/117] feat(http): validate API responses at runtime with valibot instead of blind casts (#266) --- src/lib/http.ts | 101 ++++++++++-- src/lib/response-schemas.test.ts | 104 ++++++++++++ src/lib/response-schemas.ts | 271 +++++++++++++++++++++++++++++++ 3 files changed, 461 insertions(+), 15 deletions(-) create mode 100644 src/lib/response-schemas.test.ts create mode 100644 src/lib/response-schemas.ts diff --git a/src/lib/http.ts b/src/lib/http.ts index 21369de..34a58ec 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -1,7 +1,16 @@ import { randomUUID } from 'node:crypto'; +import * as v from 'valibot'; import type { ErrorCode } from './errors.js'; import { ApiError, InterruptError, RequestTimeoutError, TransportError } from './errors.js'; import { VERSION } from '../version.js'; +import { + BATCH_RERUN_RESPONSE_SCHEMA, + BATCH_RUN_FRESH_RESPONSE_SCHEMA, + LIST_RUNS_RESPONSE_SCHEMA, + RERUN_RESPONSE_SCHEMA, + RUN_RESPONSE_SCHEMA, + TRIGGER_RUN_RESPONSE_SCHEMA, +} from './response-schemas.js'; import type { TriggerRunBody, TriggerRunResponse, @@ -108,10 +117,26 @@ export interface HttpClientOptions { shutdownSignal?: AbortSignal; } -export interface RequestOptions { +export interface RequestOptions { query?: Record; signal?: AbortSignal; requestId?: string; + /** + * Optional valibot schema for the parsed 2xx response body (issue #102). + * + * When present, `requestWithMeta` runs `v.safeParse` on the OK-path JSON: + * success returns the parsed output (unknown extra keys preserved via + * `looseObject`); failure throws an INTERNAL `ApiError` envelope naming the + * request path and the first {@link MAX_SCHEMA_ISSUES_IN_DETAILS} mismatched + * field paths (never the body itself). When absent, behavior is unchanged: + * the body is returned via the historical blind `as T` cast. + * + * Wired by the typed run helpers only (`triggerRun`, `triggerRunWithMeta`, + * `triggerRerun`, `triggerBatchRerun`, `triggerBatchRunFresh`, `getRun`, + * `listTestRuns`); generic `get`/`post`/... callers stay opt-in. + * sourceRef: response-schemas.ts. + */ + schema?: v.GenericSchema; /** * Optional JSON body for non-GET requests. Serialized with * `JSON.stringify`; `Content-Type: application/json` is auto-attached @@ -168,6 +193,11 @@ const MAX_RATE_LIMITED_DELAY_MS = 60_000; const CONFLICT_DELAY_MS = 1000; const INTERNAL_DELAY_MS = 500; +// Cap on how many valibot issues a shape-mismatch INTERNAL envelope carries in +// `details.issues` (path + message each). Keeps the envelope readable and +// guarantees the response body itself is never echoed back to the operator. +const MAX_SCHEMA_ISSUES_IN_DETAILS = 3; + /** * Result of a successful HTTP request, including the parsed body and the * `x-request-id` that was sent (useful for surfacing in happy-path output). @@ -222,23 +252,23 @@ export class HttpClient { } } - async get(path: string, options: RequestOptions = {}): Promise { + async get(path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta('GET', path, options).then(r => r.body); } - async post(path: string, options: RequestOptions = {}): Promise { + async post(path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta('POST', path, options).then(r => r.body); } - async put(path: string, options: RequestOptions = {}): Promise { + async put(path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta('PUT', path, options).then(r => r.body); } - async patch(path: string, options: RequestOptions = {}): Promise { + async patch(path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta('PATCH', path, options).then(r => r.body); } - async delete(path: string, options: RequestOptions = {}): Promise { + async delete(path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta('DELETE', path, options).then(r => r.body); } @@ -247,23 +277,26 @@ export class HttpClient { * `requestId` and `status`, so callers can surface the requestId in * happy-path output (dogfood item 1). */ - async getWithMeta(path: string, options: RequestOptions = {}): Promise> { + async getWithMeta(path: string, options: RequestOptions = {}): Promise> { return this.requestWithMeta('GET', path, options); } - async postWithMeta(path: string, options: RequestOptions = {}): Promise> { + async postWithMeta(path: string, options: RequestOptions = {}): Promise> { return this.requestWithMeta('POST', path, options); } - async putWithMeta(path: string, options: RequestOptions = {}): Promise> { + async putWithMeta(path: string, options: RequestOptions = {}): Promise> { return this.requestWithMeta('PUT', path, options); } - async patchWithMeta(path: string, options: RequestOptions = {}): Promise> { + async patchWithMeta(path: string, options: RequestOptions = {}): Promise> { return this.requestWithMeta('PATCH', path, options); } - async deleteWithMeta(path: string, options: RequestOptions = {}): Promise> { + async deleteWithMeta( + path: string, + options: RequestOptions = {}, + ): Promise> { return this.requestWithMeta('DELETE', path, options); } @@ -282,6 +315,7 @@ export class HttpClient { body, headers: { 'idempotency-key': options.idempotencyKey }, signal: options.signal, + schema: TRIGGER_RUN_RESPONSE_SCHEMA, // 409 on POST /runs means "another run is already in flight" — a // persistent condition, not a transient snapshot conflict. Retrying // would enqueue a second run once the first finishes. @@ -310,6 +344,7 @@ export class HttpClient { body, headers: { 'idempotency-key': options.idempotencyKey }, signal: options.signal, + schema: TRIGGER_RUN_RESPONSE_SCHEMA, retryOnConflict: false, // Default true: single `test run` / `test create --run` retain 429 retry. // Batch call site passes false to keep outer-loop as sole rate-limit owner. @@ -334,6 +369,7 @@ export class HttpClient { body, headers: { 'idempotency-key': options.idempotencyKey }, signal: options.signal, + schema: RERUN_RESPONSE_SCHEMA, retryOnConflict: false, }).then(r => r.body); } @@ -353,6 +389,7 @@ export class HttpClient { body, headers: { 'idempotency-key': options.idempotencyKey }, signal: options.signal, + schema: BATCH_RERUN_RESPONSE_SCHEMA, retryOnConflict: false, }).then(r => r.body); } @@ -373,6 +410,7 @@ export class HttpClient { body, headers: { 'idempotency-key': options.idempotencyKey }, signal: options.signal, + schema: BATCH_RUN_FRESH_RESPONSE_SCHEMA, retryOnConflict: false, }).then(r => r.body); } @@ -391,7 +429,10 @@ export class HttpClient { if (query.pageSize !== undefined) q.pageSize = query.pageSize; if (query.source !== undefined) q.source = query.source; if (query.since !== undefined) q.since = query.since; - return this.get(`/tests/${encodeURIComponent(testId)}/runs`, { query: q }); + return this.get(`/tests/${encodeURIComponent(testId)}/runs`, { + query: q, + schema: LIST_RUNS_RESPONSE_SCHEMA, + }); } /** @@ -421,6 +462,7 @@ export class HttpClient { return this.get(`/runs/${encodeURIComponent(runId)}`, { query: Object.keys(query).length > 0 ? query : undefined, signal: options?.signal, + schema: RUN_RESPONSE_SCHEMA, }); } @@ -481,7 +523,7 @@ export class HttpClient { async requestWithMeta( method: string, path: string, - options: RequestOptions = {}, + options: RequestOptions = {}, ): Promise> { if (!this.apiKey) throw ApiError.authRequired(); @@ -594,8 +636,9 @@ export class HttpClient { requestId, durationMs, }); + let raw: unknown; try { - return { body: (await response.json()) as T, requestId, status: response.status }; + raw = await response.json(); } catch (err) { // Interrupt passthrough (see the fetch catch above). if (err instanceof InterruptError) throw err; @@ -609,6 +652,34 @@ export class HttpClient { // and break the --output json envelope contract. throw malformedResponseError(response, requestId, err); } + if (options.schema !== undefined) { + const parsed = v.safeParse(options.schema, raw); + if (!parsed.success) { + const issues = parsed.issues.slice(0, MAX_SCHEMA_ISSUES_IN_DETAILS).map(issue => ({ + path: v.getDotPath(issue) ?? '(root)', + message: issue.message, + })); + // Shape drift is a server-side contract break: surface a typed + // INTERNAL envelope (requestId + the first mismatched paths, + // never the body) instead of letting a blind cast poison + // downstream output with undefined fields or a raw TypeError. + throw ApiError.fromEnvelope( + { + error: { + code: 'INTERNAL', + message: `Response shape mismatch from ${shortPath(path)}.`, + nextAction: + 'Retry; if it persists, report this requestId (the server returned an unexpected shape).', + requestId, + details: { issues }, + }, + }, + response.status, + ); + } + return { body: parsed.output as T, requestId, status: response.status }; + } + return { body: raw as T, requestId, status: response.status }; } let rawBody: unknown; @@ -790,7 +861,7 @@ export class HttpClient { * `client.request(...)` directly. New callers should use * `requestWithMeta` or the typed helpers (`get`, `post`, etc.). */ - async request(method: string, path: string, options: RequestOptions = {}): Promise { + async request(method: string, path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta(method, path, options).then(r => r.body); } } diff --git a/src/lib/response-schemas.test.ts b/src/lib/response-schemas.test.ts new file mode 100644 index 0000000..79c56d6 --- /dev/null +++ b/src/lib/response-schemas.test.ts @@ -0,0 +1,104 @@ +/** + * Dedicated tests for the response schemas (issue #102): the schemas must be + * loose (additive server fields pass), mirror nullability, and turn drift + * into a typed INTERNAL envelope at the HttpClient boundary. + */ + +import { describe, expect, it } from 'vitest'; +import * as v from 'valibot'; +import { HttpClient } from './http.js'; +import { RUN_RESPONSE_SCHEMA, TRIGGER_RUN_RESPONSE_SCHEMA } from './response-schemas.js'; + +const VALID_RUN = { + runId: 'run_1', + testId: 'test_1', + projectId: 'p_1', + userId: 'u_1', + status: 'passed', + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: null, + finishedAt: null, + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 0, completed: 0, passedCount: 0, failedCount: 0 }, +}; + +function makeClient(fetchImpl: typeof fetch): HttpClient { + return new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => Promise.resolve(), + random: () => 0, + }); +} + +describe('RUN_RESPONSE_SCHEMA', () => { + it('accepts a valid run and preserves unknown extra keys (additive drift is non-breaking)', () => { + const parsed = v.safeParse(RUN_RESPONSE_SCHEMA, { + ...VALID_RUN, + someFutureField: 'kept', + }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect((parsed.output as { someFutureField?: string }).someFutureField).toBe('kept'); + } + }); + + it('rejects a run missing a required field, naming the path', () => { + const withoutStatus: Record = { ...VALID_RUN }; + delete withoutStatus.status; + const parsed = v.safeParse(RUN_RESPONSE_SCHEMA, withoutStatus); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.issues.some(issue => v.getDotPath(issue) === 'status')).toBe(true); + } + }); +}); + +describe('HttpClient schema hook', () => { + it('getRun surfaces drift as a typed INTERNAL envelope with issue paths (never a blind cast)', async () => { + const drifted: Record = { ...VALID_RUN }; + delete drifted.status; + const fetchImpl = (async () => + new Response(JSON.stringify(drifted), { + status: 200, + headers: { 'content-type': 'application/json' }, + })) as typeof fetch; + const client = makeClient(fetchImpl); + const rejection = await client.getRun('run_1').catch((error: unknown) => error); + expect(rejection).toMatchObject({ code: 'INTERNAL' }); + const issues = (rejection as { getDetail: (key: string) => unknown }).getDetail('issues'); + expect(Array.isArray(issues)).toBe(true); + expect(JSON.stringify(issues)).toContain('status'); + }); + + it('a schemaless generic get still returns whatever JSON came back (unchanged behavior)', async () => { + const fetchImpl = (async () => + new Response(JSON.stringify({ anything: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })) as typeof fetch; + const client = makeClient(fetchImpl); + await expect(client.get('/me')).resolves.toEqual({ anything: true }); + }); +}); + +describe('TRIGGER_RUN_RESPONSE_SCHEMA', () => { + it('accepts the queued-run envelope', () => { + const parsed = v.safeParse(TRIGGER_RUN_RESPONSE_SCHEMA, { + runId: 'run_1', + status: 'queued', + enqueuedAt: '2026-06-01T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }); + expect(parsed.success).toBe(true); + }); +}); diff --git a/src/lib/response-schemas.ts b/src/lib/response-schemas.ts new file mode 100644 index 0000000..de86431 --- /dev/null +++ b/src/lib/response-schemas.ts @@ -0,0 +1,271 @@ +/** + * Valibot schemas for the run-path wire shapes (issue #102). + * + * `requestWithMeta` used to return `(await response.json()) as T` with zero + * runtime validation, so a drifted or partial server response surfaced as + * `undefined` output or an opaque TypeError deep inside a command. These + * schemas are wired (opt-in, via `RequestOptions.schema`) into the typed + * HttpClient helpers only: `triggerRun`, `triggerRunWithMeta`, `triggerRerun`, + * `triggerBatchRerun`, `triggerBatchRunFresh`, `getRun`, `listTestRuns`. + * The generic `get`/`post`/`put`/`patch`/`delete` paths stay schema-free. + * + * Resilience rules (additive server changes must never hard-fail the CLI): + * + * 1. Every object is `v.looseObject`: unknown extra keys pass validation AND + * are preserved in the output, so a new server field still reaches + * `--output json` consumers untouched. + * 2. Enum-ish string fields (`status`, `source`, `role`, step `type`, ...) are + * validated as open strings via {@link openWireLiteral}: the CLI already + * treats unknown values as open (e.g. `isTerminalStatus` returns false and + * the poll continues; renderers print the raw value), so a new server enum + * member must degrade gracefully, never reject the whole response. + * 3. REQUIRED-nullable interface fields use `v.nullish(inner, null)`: the wire + * may omit a nullable field entirely (real fixture evidence: the chained + * `test create --run` poll bodies in `test.test.ts` omit `error`), and + * every consumer already null-checks these, so absence normalizes to + * `null` instead of failing. OPTIONAL interface fields (`?`) use + * `v.optional` with NO default so presence/absence semantics that commands + * branch on (e.g. `RerunResponse.closure`, `RunResponse.steps`) survive + * validation byte-identically. + * + * Each schema is annotated `v.GenericSchema` against the + * interface it mirrors, so schema/interface drift fails `tsc` in this file. + */ +import * as v from 'valibot'; +import type { + BatchRerunResponse, + BatchRunFreshResponse, + ListRunsResponse, + RerunClosure, + RerunResponse, + RunResponse, + RunSource, + RunStatus, + TriggerRunResponse, +} from './runs.types.js'; + +/** + * Compile-time literal union, runtime open string. + * + * Keeps `InferOutput` aligned with the union declared in `runs.types.ts` + * while accepting any string on the wire, per resilience rule 2 above. + * `v.custom` is valibot's documented escape hatch for exactly this + * "caller-asserted type, custom runtime check" pattern. + */ +function openWireLiteral(): v.GenericSchema { + return v.custom(value => typeof value === 'string'); +} + +// --------------------------------------------------------------------------- +// GET /runs/{runId} +// --------------------------------------------------------------------------- + +/** Mirrors `RunStepSummary` (runs.types.ts): per-run step counters. */ +const RUN_STEP_SUMMARY_SCHEMA = v.looseObject({ + total: v.number(), + completed: v.number(), + passedCount: v.number(), + failedCount: v.number(), +}); + +/** Mirrors `RunStepDto` (runs.types.ts): one `?includeSteps=true` step row. */ +const RUN_STEP_DTO_SCHEMA = v.looseObject({ + stepIndex: v.string(), + type: openWireLiteral<'action' | 'assertion'>(), + action: v.string(), + status: v.nullish(openWireLiteral<'passed' | 'failed'>(), null), + description: v.nullish(v.string(), null), + error: v.nullish(v.string(), null), + screenshotUrl: v.nullish(v.string(), null), + htmlSnapshotUrl: v.nullish(v.string(), null), + createdAt: v.string(), +}); + +/** Mirrors `RunResponse` (runs.types.ts): `GET /api/cli/v1/runs/{runId}`. */ +export const RUN_RESPONSE_SCHEMA: v.GenericSchema = v.looseObject({ + runId: v.string(), + testId: v.string(), + projectId: v.string(), + userId: v.string(), + status: openWireLiteral(), + source: v.string(), + createdAt: v.string(), + startedAt: v.nullish(v.string(), null), + finishedAt: v.nullish(v.string(), null), + codeVersion: v.string(), + targetUrl: v.string(), + createdFrom: v.nullish(v.string(), null), + failedStepIndex: v.nullish(v.number(), null), + failureKind: v.nullish(v.string(), null), + // Loosened per fixture evidence (rule 3): several real poll bodies omit + // `error` entirely; consumers render it only when non-null. + error: v.nullish(v.string(), null), + videoUrl: v.nullish(v.string(), null), + stepSummary: RUN_STEP_SUMMARY_SCHEMA, + retryAfterSeconds: v.optional(v.number()), + // Client-synthesized Portal link (never sent by the server); tolerated so a + // future server echo cannot fail validation. + dashboardUrl: v.optional(v.string()), + // Absence means "steps not requested" and drives command branching, so no + // default is applied (rule 3, optional branch). + steps: v.optional(v.nullable(v.array(RUN_STEP_DTO_SCHEMA))), +}); + +// --------------------------------------------------------------------------- +// POST /tests/{testId}/runs +// --------------------------------------------------------------------------- + +/** Mirrors `TriggerRunResponse` (runs.types.ts): `POST /tests/{testId}/runs`. */ +export const TRIGGER_RUN_RESPONSE_SCHEMA: v.GenericSchema = + v.looseObject({ + runId: v.string(), + status: openWireLiteral<'queued'>(), + enqueuedAt: v.string(), + codeVersion: v.string(), + targetUrl: v.string(), + }); + +// --------------------------------------------------------------------------- +// POST /tests/{testId}/runs/rerun +// --------------------------------------------------------------------------- + +/** Mirrors `RerunClosureMember` (runs.types.ts): one BE closure member. */ +const RERUN_CLOSURE_MEMBER_SCHEMA = v.looseObject({ + testId: v.string(), + runId: v.string(), + role: openWireLiteral<'selected' | 'producer' | 'teardown'>(), +}); + +/** Mirrors `RerunClosure` (runs.types.ts): BE closure breakdown. */ +const RERUN_CLOSURE_SCHEMA: v.GenericSchema = v.looseObject({ + members: v.array(RERUN_CLOSURE_MEMBER_SCHEMA), + addedProducers: v.array(v.string()), + addedTeardowns: v.array(v.string()), + clearedCaptured: v.number(), +}); + +/** Mirrors `RerunResponse` (runs.types.ts): `POST /tests/{testId}/runs/rerun`. */ +export const RERUN_RESPONSE_SCHEMA: v.GenericSchema = v.looseObject({ + runId: v.string(), + status: openWireLiteral<'queued'>(), + enqueuedAt: v.string(), + codeVersion: v.string(), + autoHeal: v.boolean(), + // FE reruns omit `closure`; the CLI's `!!closure` truthy check relies on + // absent staying absent, so optional with no default (rule 3). + closure: v.optional(v.nullable(RERUN_CLOSURE_SCHEMA)), +}); + +// --------------------------------------------------------------------------- +// POST /tests/batch/rerun +// --------------------------------------------------------------------------- + +/** Mirrors `BatchRerunResponse` (runs.types.ts): `POST /tests/batch/rerun`. */ +export const BATCH_RERUN_RESPONSE_SCHEMA: v.GenericSchema = + v.looseObject({ + // Mirrors BatchRerunAccepted (runs.types.ts). + accepted: v.array( + v.looseObject({ testId: v.string(), runId: v.string(), enqueuedAt: v.string() }), + ), + // Mirrors BatchRerunDeferred (runs.types.ts). + deferred: v.array(v.looseObject({ testId: v.string(), reason: v.string() })), + // Mirrors BatchRerunConflict (runs.types.ts). + conflicts: v.array(v.looseObject({ testId: v.string(), currentRunId: v.string() })), + // Mirrors BatchRerunClosure / BatchRerunClosureByProject (runs.types.ts). + closure: v.looseObject({ + byProject: v.array( + v.looseObject({ + projectId: v.string(), + testIds: v.array(v.string()), + addedProducers: v.array(v.string()), + addedTeardowns: v.array(v.string()), + clearedCaptured: v.number(), + }), + ), + }), + // Optional on the wire for back-compat with older backends (D2-CLI). + notFound: v.optional(v.array(v.string())), + }); + +// --------------------------------------------------------------------------- +// POST /tests/batch/run +// --------------------------------------------------------------------------- + +/** Mirrors `BatchRunFreshResponse` (runs.types.ts): `POST /tests/batch/run`. */ +export const BATCH_RUN_FRESH_RESPONSE_SCHEMA: v.GenericSchema = + v.looseObject({ + // Mirrors BatchRunFreshAccepted (runs.types.ts); dashboardUrl is + // client-synthesized, tolerated as optional. + accepted: v.array( + v.looseObject({ + testId: v.string(), + runId: v.string(), + enqueuedAt: v.string(), + dashboardUrl: v.optional(v.string()), + }), + ), + conflicts: v.array(v.looseObject({ testId: v.string() })), + deferred: v.array(v.looseObject({ testId: v.string() })), + skippedFrontend: v.array(v.string()), + skippedIntegration: v.array(v.looseObject({ testId: v.string() })), + }); + +// --------------------------------------------------------------------------- +// GET /tests/{testId}/runs +// --------------------------------------------------------------------------- + +/** Mirrors `RunHistoryItem` (runs.types.ts): one run-history row. */ +const RUN_HISTORY_ITEM_SCHEMA = v.looseObject({ + runId: v.string(), + status: openWireLiteral(), + source: openWireLiteral(), + isRerun: v.boolean(), + createdFrom: v.nullish(v.string(), null), + createdAt: v.string(), + startedAt: v.nullish(v.string(), null), + finishedAt: v.nullish(v.string(), null), + codeVersion: v.string(), + failureKind: v.nullish(v.string(), null), + // G1b fields: optional on the wire for back-compat with older backends. + targetUrl: v.optional(v.nullable(v.string())), + targetUrlSource: v.optional(v.nullable(openWireLiteral<'run' | 'unresolved'>())), +}); + +/** Mirrors `ListRunsResponse` (runs.types.ts): `GET /tests/{testId}/runs`. */ +export const LIST_RUNS_RESPONSE_SCHEMA: v.GenericSchema = v.looseObject({ + runs: v.array(RUN_HISTORY_ITEM_SCHEMA), + nextCursor: v.nullish(v.string(), null), + // Mirrors RunHistoryMeta (runs.types.ts): every field optional, and the + // history command reads `resp.meta.note` / `resp.meta.portalUrl` directly, + // so the container itself stays required like the interface declares. + meta: v.looseObject({ + testKind: v.optional(openWireLiteral<'frontend' | 'backend'>()), + historyStartsAt: v.optional(v.string()), + note: v.optional(v.string()), + portalUrl: v.optional(v.string()), + }), +}); + +// --------------------------------------------------------------------------- +// GET /me +// --------------------------------------------------------------------------- + +/** + * Minimal `/me` identity core shared by its consumers. `doctor` reads a + * two-field optional projection (`MeIdentity` in commands/doctor.ts) while + * `auth whoami` reads the full `MeResponse` (commands/auth.ts); this schema + * validates the common identity core so it can guard either caller, and + * `looseObject` lets the full projection (scopes, env, email, ...) pass + * through untouched. Not wired into any typed helper yet: `/me` callers use + * the generic `get`, which stays schema-free in this change. + */ +export interface MeIdentityWire { + userId?: string; + keyId?: string; +} + +/** Mirrors `MeIdentity` (commands/doctor.ts): `GET /api/cli/v1/me` core. */ +export const ME_IDENTITY_SCHEMA: v.GenericSchema = v.looseObject({ + userId: v.optional(v.string()), + keyId: v.optional(v.string()), +}); From 92ebb2fb6b9ed7b3e1e23616fc00d892e38b50f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rohit=20=F0=9F=92=AB?= <147372004+sandman-sh@users.noreply.github.com> Date: Fri, 24 Jul 2026 03:41:25 +0530 Subject: [PATCH 107/117] fix(browser): catch malformed URL TypeError and map to exit 5 VALIDATION_ERROR (#274) --- src/lib/browser.test.ts | 11 +++++++++-- src/lib/browser.ts | 7 ++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/lib/browser.test.ts b/src/lib/browser.test.ts index 3d6717b..a5cb0ee 100644 --- a/src/lib/browser.test.ts +++ b/src/lib/browser.test.ts @@ -54,9 +54,16 @@ describe('openInBrowser', () => { expect(exec).not.toHaveBeenCalled(); }); - it('throws on a malformed URL', () => { + it('refuses a malformed URL with exit 5 before spawning', () => { const exec = vi.fn(); - expect(() => openInBrowser('not a url', { exec })).toThrow(); + let error: unknown; + try { + openInBrowser('not a url', { exec }); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiError); + expect((error as ApiError).exitCode).toBe(5); expect(exec).not.toHaveBeenCalled(); }); diff --git a/src/lib/browser.ts b/src/lib/browser.ts index ccd2b8a..4d25823 100644 --- a/src/lib/browser.ts +++ b/src/lib/browser.ts @@ -23,7 +23,12 @@ export interface OpenInBrowserDeps { } export function openInBrowser(url: string, deps: OpenInBrowserDeps = {}): void { - const parsed = new URL(url); + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw localValidationError('url', 'must be a valid http(s) URL', undefined, 'field'); + } if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { // User-input error, not an internal failure: classify as VALIDATION_ERROR // so it maps to exit 5 (like every other bad-argument path), not exit 1. From 26821e15a4dc03344bb3f62485b191ae00669565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rohit=20=F0=9F=92=AB?= <147372004+sandman-sh@users.noreply.github.com> Date: Fri, 24 Jul 2026 03:51:38 +0530 Subject: [PATCH 108/117] fix(completion): cross-platform shell auto-detection for Windows paths and .exe extensions (#273) * fix(completion): cross-platform shell auto-detection for Windows paths and .exe extensions * chore: trigger PR triage gate re-run --- src/commands/completion.test.ts | 8 ++++++++ src/commands/completion.ts | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/commands/completion.test.ts b/src/commands/completion.test.ts index 8388a09..c1bbdcf 100644 --- a/src/commands/completion.test.ts +++ b/src/commands/completion.test.ts @@ -23,8 +23,16 @@ describe('isShell / detectShell', () => { expect(detectShell({ SHELL: '/usr/local/bin/fish' })).toBe('fish'); }); + it('detects shells with Windows backslashes, .exe suffixes, and uppercase paths', () => { + expect(detectShell({ SHELL: 'C:\\Program Files\\Git\\bin\\bash.exe' })).toBe('bash'); + expect(detectShell({ SHELL: 'C:\\tools\\zsh.exe' })).toBe('zsh'); + expect(detectShell({ SHELL: '/usr/bin/FISH.EXE' })).toBe('fish'); + expect(detectShell({ SHELL: 'C:/Program Files\\Git\\bin/bash.exe' })).toBe('bash'); + }); + it('returns undefined for an unknown or missing shell', () => { expect(detectShell({ SHELL: '/bin/sh' })).toBeUndefined(); + expect(detectShell({ SHELL: 'C:\\Windows\\System32\\cmd.exe' })).toBeUndefined(); expect(detectShell({})).toBeUndefined(); }); }); diff --git a/src/commands/completion.ts b/src/commands/completion.ts index 15cf7a4..42668fa 100644 --- a/src/commands/completion.ts +++ b/src/commands/completion.ts @@ -42,7 +42,8 @@ export function isShell(value: string): value is Shell { /** Best-effort shell detection from `$SHELL` (e.g. "/bin/zsh" -> "zsh"). */ export function detectShell(env: NodeJS.ProcessEnv): Shell | undefined { const shellPath = env.SHELL ?? ''; - const base = shellPath.slice(shellPath.lastIndexOf('/') + 1); + const rawBase = shellPath.split(/[/\\]/).pop() ?? ''; + const base = rawBase.toLowerCase().replace(/\.exe$/, ''); return isShell(base) ? base : undefined; } From fe07bc9a18799e6b6b5400d421cc03ccfb8770e0 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:24:09 -0400 Subject: [PATCH 109/117] feat(test): GitHub-native CI output for "test run --all" (--gh-output, --summary-file) (#264) * feat(test): attach GitHub-native CI output to "test run --all" (--gh-output, --summary-file) * test(run): isolate batch-run specs from the CI runner env (GITHUB_ACTIONS leak) * fix(run): require --wait for CI-output flags, keep JSON stdout clean, harden the payload reducer --- src/commands/test.run.spec.ts | 127 ++++++++++++++- src/commands/test.ts | 70 +++++++++ src/lib/gh-output.test.ts | 146 ++++++++++++++++++ src/lib/gh-output.ts | 139 +++++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 7 + 5 files changed, 488 insertions(+), 1 deletion(-) create mode 100644 src/lib/gh-output.test.ts create mode 100644 src/lib/gh-output.ts diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 7eb8eb0..6c8be5f 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -5,7 +5,7 @@ * sleep injection is wired through `TestDeps.sleep` to avoid real delays. */ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; @@ -2725,6 +2725,7 @@ describe('runTestRunAll — batch fresh run', () => { fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, sleep: instantSleep, }, ); @@ -3458,6 +3459,7 @@ describe('[B-E2E-01] runTestRunAll --wait: non-passed runs must exit 1 (regressi fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, sleep: instantSleep, }, ); @@ -3866,6 +3868,7 @@ describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out p fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, sleep: instantSleep, }, ).catch(e => e); @@ -3957,3 +3960,125 @@ describe('runTestRun --wait — InterruptError graceful detach (DEV-331)', () => expect(stderrBlock).toContain('testsprite test wait run_abc'); }); }); + +describe('gh-output integration on run --all --wait (issue #99 reshape)', () => { + function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { + return { + runId, + testId, + projectId: 'project_be', + userId: 'user_1', + status: status as RunResponse['status'], + source: 'cli', + createdAt: '2026-06-09T11:00:00.000Z', + startedAt: '2026-06-09T11:00:01.000Z', + finishedAt: '2026-06-09T11:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: status === 'passed' ? 3 : 0, + failedCount: 0, + }, + }; + } + + function mixedHarness() { + const { credentialsPath } = makeCreds(); + const mixedBatch: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_p', runId: 'run_p', enqueuedAt: '2026-06-09T11:00:00.000Z' }, + { testId: 'test_f', runId: 'run_f', enqueuedAt: '2026-06-09T11:00:02.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: mixedBatch }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_p') return { body: makeTerminalRun('run_p', 'test_p', 'passed') }; + if (runId === 'run_f') return { body: makeTerminalRun('run_f', 'test_f', 'failed') }; + return errorBody('NOT_FOUND'); + }); + return { credentialsPath, fetchImpl }; + } + + it('under Actions with --output json: stdout stays parseable JSON, ::error:: goes to stderr', async () => { + const { credentialsPath, fetchImpl } = mixedHarness(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const err = await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + env: { GITHUB_ACTIONS: 'true' } as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + // The documented machine envelope must remain parseable as-is. + const payload = JSON.parse(stdoutLines.join('\n')) as { accepted?: unknown[] }; + expect(Array.isArray(payload.accepted)).toBe(true); + expect(stdoutLines.some(line => line.startsWith('::error'))).toBe(false); + const annotations = stderrLines.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(1); + expect(annotations[0]).toContain('test_f'); + }); + + it('--gh-output --summary-file writes the reduced artifact even though the gate exits 1', async () => { + const { credentialsPath, fetchImpl } = mixedHarness(); + const dir = mkdtempSync(join(tmpdir(), 'cli-gh-output-')); + const summaryFile = join(dir, 'summary.json'); + const stdoutLines: string[] = []; + const err = await runTestRunAll( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + ghOutput: true, + summaryFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + const artifact = JSON.parse(readFileSync(summaryFile, 'utf8')) as { + total: number; + passed: number; + failed: number; + runs: unknown[]; + }; + expect(artifact).toMatchObject({ total: 2, passed: 1, failed: 1 }); + // Forced annotations (off-Actions) land on the text stdout, not the file. + expect(stdoutLines.some(line => line.startsWith('::error'))).toBe(true); + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index ca6801c..6cb692b 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1,9 +1,11 @@ import { + appendFileSync, createWriteStream, existsSync, readFileSync, readdirSync, statSync, + writeFileSync, type WriteStream, } from 'node:fs'; import { rename, stat, unlink } from 'node:fs/promises'; @@ -92,6 +94,7 @@ import { import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; +import { emitGithubOutputs, summarizeAcceptedPayload } from '../lib/gh-output.js'; import { loadConfig } from '../lib/config.js'; import { flakyExitCode, @@ -6363,6 +6366,10 @@ interface RunTestRunAllOptions extends CommonOptions { reportFile?: string; /** --report-suite-name: optional override for the JUnit . */ reportSuiteName?: string; + /** --gh-output: force the GitHub-native output layer even off-Actions (issue #99). */ + ghOutput?: boolean; + /** --summary-file: also write the reduced machine summary JSON to this path. */ + summaryFile?: string; } async function writeBatchJUnitReportIfRequested( @@ -6927,6 +6934,41 @@ export async function runTestRunAll( }; await writeBatchJUnitReportIfRequested(opts, freshRunResults); out.print(jsonPayload); + // CI-native output layer (issue #99): emitted before the gate throws below so + // the artifacts land even when the batch exits non-zero. The summary file is a + // machine artifact written regardless of --output mode; stdout stays owned by + // the envelope above (plus Actions workflow commands, which Actions parses). + { + const env = deps.env ?? process.env; + const ghEnabled = opts.ghOutput === true || env.GITHUB_ACTIONS === 'true'; + if (ghEnabled || opts.summaryFile !== undefined) { + const ciSummary = summarizeAcceptedPayload(JSON.stringify(jsonPayload)); + if (opts.summaryFile !== undefined) { + try { + writeFileSync(opts.summaryFile, `${JSON.stringify(ciSummary, null, 2)}\n`, 'utf8'); + } catch { + stderrFn(`[run] could not write --summary-file ${opts.summaryFile}; continuing`); + } + } + if (ghEnabled) { + const stdoutFn = deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)); + emitGithubOutputs( + ciSummary, + env, + { + stdout: stdoutFn, + stderr: stderrFn, + appendFile: (path: string, content: string) => appendFileSync(path, content, 'utf8'), + // Under --output json the envelope above owns stdout; workflow + // commands go to stderr instead (the Actions runner parses both + // streams), keeping the documented machine output parseable. + annotations: opts.output === 'json' ? stderrFn : stdoutFn, + }, + { force: opts.ghOutput === true }, + ); + } + } + } // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7), // mirroring `test rerun --all`. Checked before the failed-run throw so the @@ -9125,6 +9167,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--report-suite-name ', 'optional JUnit override (default: testsprite:)', ) + .option( + '--gh-output', + 'with --all --wait: emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true', + ) + .option( + '--summary-file ', + 'with --all --wait: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file', + ) .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + @@ -9173,6 +9223,22 @@ export function createTestCommand(deps: TestDeps = {}): Command { wait: cmdOpts.wait === true, batchPath: isAll, }); + // --gh-output / --summary-file reduce the terminal batch envelope, which + // only exists on the --all --wait path (without --wait the command returns + // after enqueueing). Anywhere else they would silently no-op — reject + // loudly (same rule as --filter and the JUnit report flags). + if (cmdOpts.ghOutput === true && (!isAll || cmdOpts.wait !== true)) { + throw localValidationError( + 'gh-output', + '--gh-output only applies with --all --wait (it reduces the terminal batch envelope). Remove --gh-output, or add --all --wait.', + ); + } + if (cmdOpts.summaryFile !== undefined && (!isAll || cmdOpts.wait !== true)) { + throw localValidationError( + 'summary-file', + '--summary-file only applies with --all --wait (it reduces the terminal batch envelope). Remove --summary-file, or add --all --wait.', + ); + } if (isAll) { // --all path: wave-ordered fresh batch run. @@ -9207,6 +9273,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { report, reportFile: cmdOpts.reportFile, reportSuiteName: cmdOpts.reportSuiteName, + ghOutput: cmdOpts.ghOutput === true, + summaryFile: cmdOpts.summaryFile, }, deps, ); @@ -9687,6 +9755,8 @@ interface RunFlagOpts { report?: string; reportFile?: string; reportSuiteName?: string; + ghOutput?: boolean; + summaryFile?: string; } interface WaitFlagOpts { diff --git a/src/lib/gh-output.test.ts b/src/lib/gh-output.test.ts new file mode 100644 index 0000000..759ecaa --- /dev/null +++ b/src/lib/gh-output.test.ts @@ -0,0 +1,146 @@ +/** + * Unit tests for the CI-native output layer attached to `test run --all` + * (issue #99, reshaped from the withdrawn top-level `ci` command). The heavy + * lifting (trigger + poll) is the batch command's, already covered by its own + * suites; these tests cover the presentation seams: payload reduction, the + * job-summary Markdown, and the GitHub gating (env-driven and `--gh-output` + * forced). + */ + +import { describe, expect, it } from 'vitest'; +import { + emitGithubOutputs, + renderJobSummaryMarkdown, + summarizeAcceptedPayload, + type CiSummary, +} from './gh-output.js'; + +const PAYLOAD = JSON.stringify({ + accepted: [ + { + testId: 'test_a', + runId: 'run_a', + status: 'passed', + dashboardUrl: 'https://portal.example.com/a', + }, + { + testId: 'test_b', + runId: 'run_b', + status: 'failed', + error: { code: 'INTERNAL', message: 'boom', exitCode: 1 }, + }, + { testId: 'test_c', runId: 'run_c', status: 'timeout' }, + ], + conflicts: [], +}); + +describe('summarizeAcceptedPayload', () => { + it('reduces accepted[] rows into counts and rows', () => { + const summary = summarizeAcceptedPayload(PAYLOAD); + expect(summary).toMatchObject({ total: 3, passed: 1, failed: 1, timedOut: 1 }); + expect(summary.runs[1]).toMatchObject({ testId: 'test_b', status: 'failed', error: 'boom' }); + }); + + it('unparseable or non-batch output reduces to an empty summary (never throws)', () => { + expect(summarizeAcceptedPayload('')).toMatchObject({ total: 0, passed: 0 }); + expect(summarizeAcceptedPayload('{"method":"POST"}')).toMatchObject({ total: 0 }); + expect(summarizeAcceptedPayload('not json')).toMatchObject({ total: 0 }); + }); + + it('valid-JSON non-record payloads and null rows are skipped, not crashes', () => { + expect(summarizeAcceptedPayload('null')).toMatchObject({ total: 0 }); + expect(summarizeAcceptedPayload('"a string"')).toMatchObject({ total: 0 }); + expect(summarizeAcceptedPayload('[1,2]')).toMatchObject({ total: 0 }); + const mixed = summarizeAcceptedPayload( + JSON.stringify({ accepted: [null, 42, { testId: 'test_ok', status: 'passed' }] }), + ); + expect(mixed.total).toBe(1); + expect(mixed.runs[0]).toMatchObject({ testId: 'test_ok', status: 'passed' }); + }); +}); + +describe('renderJobSummaryMarkdown', () => { + it('renders the counts headline and one table row per run', () => { + const md = renderJobSummaryMarkdown(summarizeAcceptedPayload(PAYLOAD)); + expect(md).toContain('**1/3 passed** (1 failed, 1 timed out)'); + expect(md).toContain('| test_a | passed | [dashboard](https://portal.example.com/a) |'); + expect(md).toContain('| test_c | timeout | run_c |'); + }); +}); + +describe('emitGithubOutputs', () => { + const summary: CiSummary = summarizeAcceptedPayload(PAYLOAD); + + function makeSinks() { + const stdout: string[] = []; + const stderr: string[] = []; + const appended: Array<{ path: string; content: string }> = []; + return { + stdout, + stderr, + appended, + sinks: { + stdout: (line: string) => stdout.push(line), + stderr: (line: string) => stderr.push(line), + appendFile: (path: string, content: string) => appended.push({ path, content }), + }, + }; + } + + it('appends the job summary and annotates only non-passed runs under Actions', () => { + const { stdout, appended, sinks } = makeSinks(); + emitGithubOutputs( + summary, + { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: '/gh/summary.md' }, + sinks, + ); + expect(appended).toHaveLength(1); + expect(appended[0]!.path).toBe('/gh/summary.md'); + expect(appended[0]!.content).toContain('TestSprite results'); + const annotations = stdout.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(2); + expect(annotations[0]).toContain('test_b'); + expect(annotations[0]).toContain('boom'); + expect(annotations[1]).toContain('test_c'); + }); + + it('emits nothing off-CI, and a broken summary file downgrades to stderr', () => { + const offCi = makeSinks(); + emitGithubOutputs(summary, {}, offCi.sinks); + expect(offCi.stdout).toHaveLength(0); + expect(offCi.appended).toHaveLength(0); + + const broken = makeSinks(); + emitGithubOutputs( + summary, + { GITHUB_STEP_SUMMARY: '/gh/summary.md' }, + { + ...broken.sinks, + appendFile: () => { + throw new Error('EROFS'); + }, + }, + ); + expect(broken.stderr.join('\n')).toContain('could not append'); + }); + + it('force (--gh-output) emits annotations off-Actions; the step summary still needs its env path', () => { + const forced = makeSinks(); + emitGithubOutputs(summary, {}, forced.sinks, { force: true }); + const annotations = forced.stdout.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(2); + expect(forced.appended).toHaveLength(0); + }); + + it('a dedicated annotations sink diverts workflow commands off the primary stdout', () => { + const { stdout, sinks } = makeSinks(); + const diverted: string[] = []; + emitGithubOutputs( + summary, + { GITHUB_ACTIONS: 'true' }, + { ...sinks, annotations: line => diverted.push(line) }, + ); + expect(stdout).toHaveLength(0); + expect(diverted.filter(line => line.startsWith('::error'))).toHaveLength(2); + }); +}); diff --git a/src/lib/gh-output.ts b/src/lib/gh-output.ts new file mode 100644 index 0000000..ea9b379 --- /dev/null +++ b/src/lib/gh-output.ts @@ -0,0 +1,139 @@ +/** + * CI-native output layer for the batch run path (issue #99, reshaped from + * the withdrawn top-level `ci` command per the #264 review). + * + * `test run --all --wait` presents its result in the formats CI consumes: + * (a) a stable machine summary `{total, passed, failed, timedOut, runs[]}` + * written to `--summary-file ` when requested, + * (b) a Markdown results table appended to `$GITHUB_STEP_SUMMARY` when + * running under GitHub Actions, + * (c) one `::error::` workflow-command line per non-passed run so failures + * annotate the PR checks tab. + * Activation: `GITHUB_ACTIONS=true` in the environment, or the explicit + * `--gh-output` flag (which forces the annotations even off-Actions, so the + * behavior is previewable locally). All writes are best-effort: a broken + * summary file must never mask the batch gate's exit code. + */ + +export interface CiRunRow { + testId: string; + runId?: string; + status: string; + dashboardUrl?: string; + error?: string; +} + +export interface CiSummary { + total: number; + passed: number; + failed: number; + timedOut: number; + runs: CiRunRow[]; +} + +/** + * Reduce the batch command's JSON payload into the CI summary. The parse is + * defensive: it reads the same `accepted[]` rows the automation contract + * documents, and anything unparseable (dry-run envelope, partial output + * after a timeout) reduces to an empty run list rather than a crash. + */ +export function summarizeAcceptedPayload(capturedJson: string): CiSummary { + let parsed: unknown; + try { + parsed = JSON.parse(capturedJson); + } catch { + // Not JSON at all (dry-run banner path or truncated output): no rows. + parsed = undefined; + } + // `JSON.parse('null')` and non-object payloads are valid JSON but carry no + // batch envelope — treat them like unparseable input instead of crashing. + const payload: { accepted?: unknown } = + parsed !== null && typeof parsed === 'object' ? (parsed as { accepted?: unknown }) : {}; + const rows: CiRunRow[] = Array.isArray(payload.accepted) + ? payload.accepted + .filter( + (entry): entry is Record => entry !== null && typeof entry === 'object', + ) + .map(row => { + const errorMessage = + row.error !== null && typeof row.error === 'object' + ? (row.error as { message?: unknown }).message + : undefined; + return { + testId: String(row.testId ?? ''), + ...(typeof row.runId === 'string' ? { runId: row.runId } : {}), + status: String(row.status ?? 'unknown'), + ...(typeof row.dashboardUrl === 'string' ? { dashboardUrl: row.dashboardUrl } : {}), + ...(typeof errorMessage === 'string' ? { error: errorMessage } : {}), + }; + }) + : []; + const passed = rows.filter(row => row.status === 'passed').length; + const timedOut = rows.filter(row => row.status === 'timeout').length; + const failed = rows.length - passed - timedOut; + return { total: rows.length, passed, failed, timedOut, runs: rows }; +} + +/** Markdown table for the GitHub job summary. */ +export function renderJobSummaryMarkdown(summary: CiSummary): string { + return [ + '## TestSprite results', + '', + `**${summary.passed}/${summary.total} passed** (${summary.failed} failed, ${summary.timedOut} timed out)`, + '', + '| Test | Status | Run |', + '| --- | --- | --- |', + ...summary.runs.map( + row => + `| ${row.testId} | ${row.status} | ${ + row.dashboardUrl ? `[dashboard](${row.dashboardUrl})` : (row.runId ?? '') + } |`, + ), + '', + ].join('\n'); +} + +/** + * Emit the GitHub-native surfaces. Self-gating on the standard env vars: + * `$GITHUB_STEP_SUMMARY` (a file path Actions provides) receives the Markdown + * table; `GITHUB_ACTIONS=true` enables one `::error::` workflow command per + * non-passed run on stdout (Actions parses workflow commands from stdout). + * `force` (the `--gh-output` flag) emits the annotations even off-Actions; + * the step summary still requires the env-provided file path to exist. + * Both writes are best-effort: a broken summary file must not mask the gate. + */ +export function emitGithubOutputs( + summary: CiSummary, + env: NodeJS.ProcessEnv, + sinks: { + stdout: (line: string) => void; + stderr: (line: string) => void; + appendFile: (path: string, content: string) => void; + /** + * Where `::error::` workflow-command lines go. Defaults to `stdout`; the + * caller passes stderr under `--output json` so the machine envelope on + * stdout stays parseable (the Actions runner processes workflow commands + * on both streams). + */ + annotations?: (line: string) => void; + }, + opts: { force?: boolean } = {}, +): void { + const summaryPath = env.GITHUB_STEP_SUMMARY; + if (typeof summaryPath === 'string' && summaryPath.length > 0) { + try { + sinks.appendFile(summaryPath, renderJobSummaryMarkdown(summary)); + } catch { + sinks.stderr('[run] could not append to GITHUB_STEP_SUMMARY; continuing'); + } + } + if (env.GITHUB_ACTIONS === 'true' || opts.force === true) { + const annotate = sinks.annotations ?? sinks.stdout; + for (const row of summary.runs) { + if (row.status === 'passed') continue; + const detail = row.error !== undefined ? ` ${row.error}` : ''; + const link = row.dashboardUrl !== undefined ? ` ${row.dashboardUrl}` : ''; + annotate(`::error title=TestSprite ${row.testId}::status=${row.status}${detail}${link}`); + } + } +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 55a7252..bdee849 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -679,6 +679,13 @@ Options: --report-file output path for --report (atomic write) --report-suite-name optional JUnit override (default: testsprite:) + --gh-output with --all --wait: emit GitHub-native output + (::error:: annotations per non-passed run; + job-summary table when $GITHUB_STEP_SUMMARY is + set). Auto-enabled when GITHUB_ACTIONS=true + --summary-file with --all --wait: also write the reduced machine + summary JSON {total, passed, failed, timedOut, + runs[]} to this file -h, --help display help for command Dependency-aware fresh run (M4): From 8e79ee0fcfec5718b78d836d6eec8db5deec1c13 Mon Sep 17 00:00:00 2001 From: Contributor Date: Fri, 24 Jul 2026 11:25:43 +0200 Subject: [PATCH 110/117] test(rerun): restore DEV-331 InterruptError batch --wait regression Re-add the R-BAT batch rerun --wait InterruptError test removed during the TimeoutError stdout refactor. Reviewer requires no main describe/it deletions unless explicitly agreed obsolete. Co-authored-by: Cursor --- src/commands/test.rerun.spec.ts | 98 ++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index c4e865a..995b6bd 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -9,7 +9,8 @@ import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; import type { FetchImpl } from '../lib/http.js'; import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; @@ -4952,3 +4953,98 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t void fetchCallCount; }); }); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach during batch rerun --wait (SIG-6) +// --------------------------------------------------------------------------- + +describe('R-BAT: batch rerun --wait — InterruptError partial lists all dispatched runIds (DEV-331)', () => { + it('interrupt mid fan-out → stdout partial covers every accepted runId, honest stderr, exit 130', async () => { + const creds = makeCreds(); + const shutdown = new ShutdownController(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + // Batch trigger resolves; every run poll hangs until the composed signal aborts. + const fetchImpl: FetchImpl = (async (input: unknown, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/batch/rerun')) { + return new Response(JSON.stringify(batchResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as FetchImpl; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const pending = runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 10); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + + // SIG-6: the partial lists ALL dispatched runIds, marked running. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ runId: string; status: string }>; + }; + const byRunId = new Map(stdoutJson.accepted.map(r => [r.runId, r.status])); + expect(byRunId.get('run_b1')).toBe('running'); + expect(byRunId.get('run_b2')).toBe('running'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('run_b1'); + expect(stderrBlock).toContain('run_b2'); + }); +}); From b35dbaeb57add1cf666cc6e206a1662660f553e8 Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Thu, 6 Aug 2026 01:05:27 +0000 Subject: [PATCH 111/117] release: v0.5.0 Private-Snapshot-RevId: adcb0d580e3d16988822e319adcab95986709956 --- .github/workflows/ci.yml | 12 +- .github/workflows/release.yaml | 2 +- .github/workflows/test-coverage.yml | 2 +- .gitleaks.toml | 32 +- CHANGELOG.md | 23 + DOCUMENTATION.md | 132 +- README.md | 27 +- package-lock.json | 657 +++----- package.json | 21 +- schemas/plan.schema.json | 61 + src/commands/agent.test.ts | 111 ++ src/commands/agent.ts | 21 +- src/commands/auth.test.ts | 327 +++- src/commands/auth.ts | 59 + src/commands/doctor.test.ts | 69 +- src/commands/doctor.ts | 31 +- src/commands/init.test.ts | 105 +- src/commands/init.ts | 18 +- src/commands/project.test.ts | 539 +++++++ src/commands/project.ts | 251 ++- src/commands/test.flaky.spec.ts | 204 ++- .../test.rerun.closure-fanout.spec.ts | 257 ++++ src/commands/test.rerun.spec.ts | 889 +++++++++-- src/commands/test.result.history.spec.ts | 110 +- src/commands/test.run.spec.ts | 193 ++- src/commands/test.test.ts | 913 ++++++++++- src/commands/test.ts | 1344 ++++++++++++++--- src/commands/test.wait.spec.ts | 285 ++++ src/commands/usage.test.ts | 360 ++++- src/commands/usage.ts | 231 ++- src/index.ts | 188 ++- src/lib/agent-targets.ts | 13 +- src/lib/api-key-prefix-leak-guard.test.ts | 84 ++ src/lib/bundle.test.ts | 63 + src/lib/bundle.ts | 56 +- src/lib/client-factory.test.ts | 53 + src/lib/client-factory.ts | 43 +- src/lib/dry-run/samples.test.ts | 9 +- src/lib/dry-run/samples.ts | 19 +- src/lib/errors.test.ts | 120 ++ src/lib/errors.ts | 91 +- src/lib/flaky.test.ts | 27 + src/lib/flaky.ts | 24 +- src/lib/http.test.ts | 28 + src/lib/http.ts | 9 +- src/lib/org-render.test.ts | 85 ++ src/lib/org-render.ts | 80 + src/lib/plan-schema.spec.ts | 216 +++ src/lib/poll.ts | 2 +- src/lib/render-error.test.ts | 62 +- src/lib/render-error.ts | 27 + src/lib/response-schemas.test.ts | 251 ++- src/lib/response-schemas.ts | 85 +- src/lib/runs.types.ts | 62 +- src/lib/skill-nudge.test.ts | 30 + src/lib/skill-nudge.ts | 23 + src/lib/telemetry.spec.ts | 272 ++++ src/lib/telemetry.ts | 240 +++ src/lib/v3-advisory.test.ts | 12 +- src/lib/v3-advisory.ts | 16 +- src/version.ts | 2 +- test/__snapshots__/help.snapshot.test.ts.snap | 152 +- test/cli.subprocess.test.ts | 155 +- test/contract/p4-schema.test.ts | 2 +- test/contract/p5-schema.test.ts | 2 +- test/e2e/agent-install.e2e.test.ts | 76 + test/e2e/setup.e2e.test.ts | 8 +- test/e2e/signal.e2e.test.ts | 4 +- test/e2e/skill-nudge.e2e.test.ts | 16 +- test/global-setup.ts | 31 + test/help.snapshot.test.ts | 14 +- test/helpers/assertFreshBuild.ts | 42 + test/helpers/stdoutPurity.ts | 1 + test/mock-backend/handlers.smoke.test.ts | 8 +- vitest.config.ts | 8 +- 75 files changed, 8914 insertions(+), 1183 deletions(-) create mode 100644 schemas/plan.schema.json create mode 100644 src/commands/test.rerun.closure-fanout.spec.ts create mode 100644 src/lib/api-key-prefix-leak-guard.test.ts create mode 100644 src/lib/org-render.test.ts create mode 100644 src/lib/org-render.ts create mode 100644 src/lib/plan-schema.spec.ts create mode 100644 src/lib/telemetry.spec.ts create mode 100644 src/lib/telemetry.ts create mode 100644 test/global-setup.ts create mode 100644 test/helpers/assertFreshBuild.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3766b0f..a807657 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: 'npm' @@ -43,7 +43,7 @@ jobs: persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: 'npm' @@ -63,7 +63,7 @@ jobs: persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ matrix.node-version }} cache: 'npm' @@ -83,7 +83,7 @@ jobs: persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: 'npm' @@ -107,7 +107,7 @@ jobs: persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ matrix.node-version }} cache: 'npm' @@ -129,7 +129,7 @@ jobs: persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: 'npm' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8ef6dc9..aad3ffc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -10,7 +10,7 @@ jobs: id-token: write # npm provenance steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 registry-url: 'https://registry.npmjs.org' diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 0e20d9e..025c814 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -20,7 +20,7 @@ jobs: persist-credentials: false - name: Setup Node.js - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 cache: 'npm' diff --git a/.gitleaks.toml b/.gitleaks.toml index b31d49c..a98099c 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -18,7 +18,7 @@ regexes = [ # Unit-test fixtures, e.g. sk-secret-12345 in *.test.ts '''sk-secret-[0-9]+''', # Documented placeholder keys used by dry-run/sample responses - '''sk-user-(test|DRY-RUN)''', + '''sk-(user|member)-(test|DRY-RUN)''', # Example values inside internal (never-shipped, dropped-before-publish) # design docs. These are only ever scanned by ci.yml's continuous gitleaks # job (new, 2026-07) — the release-time scan @@ -39,3 +39,33 @@ regexes = [ # to contain this substring (review round 2, 2026-07-15; reproduced). '''^eyJlayI6InByb2plY3RfYTQ3YjJjMTEifQ==$''', ] + +# ── provider-specific rules ────────────────────────────────────────────────── +# gitleaks' default rule set has no pattern for TestSprite's own token shapes, +# so before these a real credential in the tree was detected only by the +# release-time LEAK_RE greps in `scripts/make-public-snapshot.sh` / +# `copybara/leak-safety-harness.sh` — and only for the `sk-user-` spelling. +# The 2026-08-01 rename to `sk-member-` made every NEWLY minted key invisible +# to all three at once. After an incident that put 452 live keys in 499 public +# repositories, "the detector does not know about the current credential" is +# the failure that only surfaces the next time. Any future prefix must be added +# here in the same change that introduces it. +[[rules]] +id = "testsprite-membership-key" +description = "TestSprite membership API key (sk-member-… / pre-rename tsp__…)" +# 43 base64url chars is the exact entropy tail both mint paths emit; the +# placeholder literals in the allowlist above are far shorter and cannot match. +# The `tsp_` arm matches the whole NAMESPACE (`tsp__`), not just `tsp_u_`: +# the CLI's own format gate accepts any `tsp_`-prefixed token, and `tsp_sa_` is +# already reserved for phase-2 service-account keys. Pinning `u_` here would +# mean the day those are minted the CLI accepts a credential no scan can see. +regex = '''\b(?:sk-member-|tsp_[a-z]{1,4}_)[A-Za-z0-9_-]{43}\b''' +keywords = ["sk-member-", "tsp_u_"] + +[[rules]] +id = "testsprite-legacy-envelope-key" +description = "TestSprite legacy envelope API key (sk-user-…)" +# The legacy envelope is variable-length base64url; 40+ keeps it clear of the +# documented `sk-user-test` / `sk-user-DRY-RUN` placeholders. +regex = '''\bsk-user-[A-Za-z0-9_-]{40,}\b''' +keywords = ["sk-user-"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 45803e1..33d23be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,29 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +## [0.5.0] - 2026-08-05 + +### Added + +- **Command-outcome telemetry — one event per invocation, opt out with one env var.** The CLI now posts a single "how did that command end" event to the TestSprite backend (`POST /api/cli/v1/telemetry`), which forwards it server-side; the CLI ships no analytics SDK and no write key. The payload is a fixed allowlist of low-cardinality fields — the leaf command that ran, `success`/`error`, the exit code, a stable machine `errorCode`, and a duration. It never carries target URLs, API keys, flag values, argument values, or error messages. Opt out with `TESTSPRITE_NO_TELEMETRY` or the cross-tool `DO_NOT_TRACK`. Nothing is sent when no API key is configured, under `--dry-run`, when no leaf command ran (bare `--help`, a parse error), on Ctrl-C, or from the purely local `completion` / `test create --plan-template` paths. The post is bounded at one second and every failure is swallowed: telemetry can never change a command's output, exit code, or duration beyond that bound. +- **`test result --history --rerun` / `--no-rerun`** — narrow run history to only reruns or only fresh runs. Applied client-side on each page's `isRerun` (the backend has no rerun filter), so — exactly like `--source` — a filtered page can come back short or empty while `nextCursor` is still set; paginate to continue. Absent means no filter, never a silent default. +- **`sk-member-` membership keys are accepted.** Workspace membership keys are now minted with an `sk-member-` prefix. The CLI's fail-fast key-format check knew only `sk-user-` and `tsp_`, so it rejected every newly minted membership key locally, before the key ever reached the server. All three families are now accepted from one list that also renders the error message, so the check and the message cannot drift apart; existing `tsp_` keys keep authenticating. +- **`project get` shows the project's target URL.** A backend project created without `--url` is dead on arrival on the V3 execution path (its first run is rejected `no-target-resolvable`), and there was no way to see that from the CLI. `project get` now prints `targetUrl`, and `(not set)` plus the exact `project update … --url` fix-it command when there is none. Absent-safe: against a backend that does not report the field the line is omitted entirely rather than claiming every project is URL-less. `project list` deliberately does not show it — resolving it per row costs one extra read per project. + +### Fixed + +- **`--wait` no longer fails a healthy V3 backend run.** A V3 run reports `targetUrl: null` (and `codeVersion: null` for a plan-driven frontend case with no code row), which the response validator rejected — so `test create --run --wait` and `test wait` printed `INTERNAL: Response shape mismatch` _after_ the run had already dispatched. The wire contract always allowed null on those fields; the CLI was the side out of spec. +- **A rate limit no longer ends a multi-run `test wait`.** With several run ids, one 429 from the backend's IP-keyed limiter (which a CI runner or NAT can trip through no fault of the key) recorded that member as a poll error and exited 7 — reporting a healthy run as a failure to observe it. Each member now re-polls up to 3 times honoring the server's `Retry-After`; every backoff is clamped to the shared `--timeout` deadline, stays interruptible by Ctrl-C, and reports a timeout rather than a rate limit if the deadline is reached during one. When a throttle outlasts the retry budget **and** nothing else went wrong, the exit code is 11 (rate limited — back off, then re-attach) instead of 7, which told an automated caller to retry immediately and walk straight back into the limiter. Any real timeout or non-passed run in the same invocation still exits 7 / 1. +- **`usage` reports the wallet that actually pays for your runs.** On an organization-billed account the spend drains the workspace wallet, so the legacy per-user balance was a number that never moved. When the backend reports an active organization, `usage` renders the workspace block (plan, remaining, per-seat allowance, seats) and suppresses the legacy balance; `auth status` gains a one-line `org:` summary. The `~N runs` estimate stays on the legacy path — workspace billing prices each action separately, so there is no single per-run rate. +- **`--no-auto-heal` was a silent no-op on rerun, and `test flaky` never disabled healing at all.** The rerun request body omitted `autoHeal` entirely when you opted out, and the server treats an absent field as heal-on — so the opt-out did nothing, and `test flaky`'s strict verbatim replays (whose whole point is that a nondeterministic pass cannot be masked by a heal) were quietly healing too. All three rerun dispatch sites now send an explicit boolean, `test flaky` always sends `autoHeal: false`, and `--dry-run` previews the same body the real request sends. Rerun and batch-rerun responses may also carry advisories now (e.g. an engine that does not yet honor the opt-out); each renders as one `[advisory]` line on stderr, deduped across batch chunks, deferred retries, and flaky probes. +- **One closure member's poll error no longer discards a whole backend `test rerun --wait`.** A backend rerun expands its dependency closure server-side and polls every member. A single member's non-timeout poll error (an API error, a transient failure, a malformed response) rejected the entire fan-out, throwing away the sibling results **and** the named test's own verdict. Classifiable member errors now land in `closureFailures[]` and the named test still prints its result; its own error, if any, is re-thrown after the payload. Any unobserved member — timed out or poll-errored — still forces exit 7, so `--wait` never reports success with an unconfirmed dependency. +- **Run cards use the server's dashboard link when it offers one.** `GET /runs/{runId}` may now carry a server-built `dashboardUrl`, and the run-completion paths prefer it over the link this process templates. Two things the server knows and the CLI cannot: which store answered the read (a project created through the V3-native write path has no row behind the route the CLI templates, so its link could not render at all), and the portal origin outside production (against any non-prod endpoint the CLI printed no link whatsoever). An absent field reopens the client computation, so an older backend behaves exactly as before. Separately, a `dashboardUrl: null` on the wire no longer fails response validation and kills `test wait` — the same trap that had to be un-sprung for a null `targetUrl`. + +### Changed + +- **`auth status` and `doctor` explain a workspace you cannot reach.** An API key belongs to exactly one workspace, chosen when it is minted. Someone in a team using a personal key previously just saw the team's projects missing from `project list`, and addressing one by id failed the same way a typo does. Both commands now name the unreachable workspaces and point at minting a key there. Silent for solo users and for keys that are already workspace-bound. +- **The V3 routing advisory only lists gaps that are still open.** It claimed `test cancel` could 404 and `test delete` could leave a zombie run; both have shipped. It now names the two real ones: `--target-url` is ignored on frontend runs, and a frontend rerun replays the run it was pointed at rather than the latest saved code. + ## [0.4.0] - 2026-07-16 ### Added diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 8704182..8fd3bbb 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -11,6 +11,7 @@ The full reference for the TestSprite CLI: install verification, manual setup, e - [Manual setup](#manual-setup) - [The complete agent loop](#the-complete-agent-loop) - [Agent onboarding (`agent install`)](#agent-onboarding-agent-install) +- [Plan file format](#plan-file-format) - [Command reference](#command-reference) - [Read commands](#read-commands) - [Write commands](#write-commands) @@ -68,6 +69,8 @@ testsprite auth whoami Credentials are stored at `~/.testsprite/credentials` (INI-style, mode `0600`). See [Configuration](#configuration) for profiles, environment overrides, and scopes. +For an org-scoped API key, `auth status` additionally prints an `orgs:` line (every organization your account belongs to) and an `org binding:` line (the specific organization this key is bound to). Both are omitted for a personal key or an older backend that doesn't report them. + ### 2. Run your first test ```bash @@ -131,6 +134,52 @@ The `codex` target uses **managed-section mode** — it writes only a sentinel-d Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. +## Plan file format + +A **plan file** is the JSON document `test create --plan-from ` ingests to author one **frontend** test (bulk-create takes the same shape, one spec per line/file — see [`test create-batch`](#testsprite-test-create-batch)). It holds exactly **ONE** test as a single JSON object — a top-level array is rejected (use `create-batch` for many). + +```json +{ + "$schema": "https://raw.githubusercontent.com/TestSprite/testsprite-cli/v0.4.0/schemas/plan.schema.json", + "projectId": "prj_abc123", + "type": "frontend", + "name": "Login rejects an empty password", + "planSteps": [ + { + "type": "action", + "description": "Navigate to /login and submit the form with an empty password" + }, + { + "type": "assertion", + "description": "Verify an inline error says the password is required" + } + ] +} +``` + +Get this exact skeleton without hand-copying it from this file: `testsprite test create --plan-template` (pure-local, prints to stdout — see [`test create`](#testsprite-test-create)). The same example is embedded in `test create --help`. **The `$schema` value above is pinned to the CLI version that generated this page (`v0.4.0`)** — `--plan-template`'s live output always pins to your actually-installed version instead, so on a later release the two will differ; run the command yourself rather than trusting this snippet's `$schema` value verbatim. + +| Field | Required | Type | Notes | +| ------------- | -------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `projectId` | yes | string | Returned by `testsprite project list`. Non-empty, not whitespace-only. | +| `type` | yes | `"frontend"` | `--plan-from` only accepts `frontend` — a `backend`-typed plan is rejected pre-flight with a `nextAction` pointing at `test create --type backend --code-file ` (backend tests are authored from a code file, not a plan). | +| `name` | yes | string | An assertable behavior statement (subject + verb + outcome), not a noun fragment. | +| `description` | no | string | One-sentence elaboration of `name` — the condition plus the expected outcome. | +| `priority` | no | `"p0"` \| `"p1"` \| `"p2"` \| `"p3"` | p0 = must-pass, p1 = important paths, p2 = edge cases, p3 = cosmetic. | +| `planSteps` | yes | `Array<{ type: "action" \| "assertion", description: string }>` | **1–200 steps**, describing user intent in plain language, not selectors. | + +**Size cap:** the whole file must be **≤ 256 KB** (`test create-batch` caps the aggregate batch at 5 MB / 50 specs). Both caps are enforced client-side before any network call. + +**`{{...}}`-style placeholders are NOT substituted.** The CLI does no variable substitution — a step like `"description": "log in as {{LOGIN_USER}}"` is structurally valid (it validates and creates fine) but the browser agent types the literal braces into the field. `test create --plan-from` prints a non-fatal `[advisory]` when it detects one; store login credentials on the project instead: `testsprite project update --username --password `, or Portal → Project Settings. + +**`$schema` for live editor validation:** the optional `"$schema"` key above (an ordinary extra property — the CLI does not restrict a plan file to a fixed property set, so a non-string value there is exactly as valid as a string one) points VS Code's JSON language service — and by extension Copilot's inline completions — at [`schemas/plan.schema.json`](./schemas/plan.schema.json), shipped in both this repo and the npm package. Point it at a local copy instead (`node_modules/@testsprite/testsprite-cli/schemas/plan.schema.json`) if you'd rather not depend on the network URL resolving in your editor. This value is **version-pinned** (`v`, not `main`) — a plan authored against one CLI version keeps resolving the SAME schema later, even after `main` gains new required fields. + +**Machine-readable ground truth**, for tooling that wants to fetch the contract instead of parsing this page: [`schemas/plan.schema.json`](./schemas/plan.schema.json) — this schema is the ground truth for the `--plan-from` **command** end-to-end (e.g. it restricts `type` to `"frontend"` only, matching what actually succeeds, not `assertPlanShape`'s looser raw structural check in isolation); if the schema and the validator ever disagree, the validator's real acceptance behavior is authoritative and the schema is out of date. The schema file's own internal `$id` intentionally stays pinned to the canonical `main` URL — `$id` is the schema's IDENTITY (what it calls itself for cross-referencing), not a fetch instruction, so it does not need version-pinning the way the `$schema` fetch hint above does. + +**Multiple tests?** Draft a `plans.jsonl` (one plan object per line) or a directory of `*.json` plan files, then `test create-batch --plans ` / `--plan-from-dir

`. Max 50 specs / 5 MB per batch. + +**Zero-cost iteration loop:** `test create --plan-from --dry-run` runs the exact same local validation as a real create — no network call, no auth, no credits spent — so an agent (or you) can iterate on a plan file until it validates before ever hitting the API. `test lint` runs the same validators across a whole batch, collecting every problem instead of stopping at the first. + ## Command reference Every command supports the [global flags](#global-flags), and every example below pairs a real call with a `--dry-run` companion that works on a fresh install with no auth. @@ -152,6 +201,8 @@ Common flags: - `--starting-token ` — opaque cursor from a previous response. - `--max-items ` — client-side cap on total items across auto-paged pages. +For an org-scoped API key, the text table gains an `ORG` column (project owning organization) whenever at least one row carries org attribution; a personal key or a page with no org data keeps the legacy column set unchanged. `--output json` always includes `orgId`/`orgName` when the backend supplies them. + #### `testsprite project get ` Get a single project by id. Project ids look like `proj_xxxxxxxx` and come from `project list`. @@ -268,7 +319,7 @@ testsprite test failure summary test_xxxxxxxx --dry-run --output json ### Write commands -Require the `write:tests` scope (project commands require `write:projects`), except `test scaffold` and `test lint`, which are pure-local authoring helpers — no network, no credentials, no scope. +Require the `write:tests` scope (project commands require `write:projects`), except `test scaffold`, `test lint`, and `test create --plan-template`, which are pure-local authoring helpers — no network, no credentials, no scope. #### `testsprite test scaffold` @@ -293,7 +344,9 @@ testsprite test lint --steps ./refined.plan.json # the shape `test plan pu #### `testsprite test create` -Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. Backend tests can declare wave-ordering dependencies at create time — `--produces ` / `--needs ` (repeatable) and `--category ` — and amend them later via `test update`. +Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from` (see [Plan file format](#plan-file-format)). With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. Backend tests can declare wave-ordering dependencies at create time — `--produces ` / `--needs ` (repeatable) and `--category ` — and amend them later via `test update`. + +`--plan-template` prints the canonical minimal plan-file skeleton to stdout and exits — pure-local, no network/credentials, ignores every other flag. The exact same example is embedded in `test create --help`. ```bash # Backend test from a code file @@ -304,6 +357,9 @@ testsprite test create --project proj_xxxxxxxx --type backend --name "Login API" testsprite test create --plan-from ./checkout.plan.json --type frontend \ --run --wait --timeout 600 --output json +# Print the plan-file skeleton, edit it, then create from it +testsprite test create --plan-template > plan.json + # Dry-run prints the canned wire envelope testsprite test create --plan-from ./checkout.plan.json --dry-run --output json ``` @@ -357,6 +413,8 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --expected-st testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --output json ``` +**V3-migrated accounts:** the backend returns `UNSUPPORTED` (exit 7, with an actionable `nextAction`) for this endpoint on accounts that have been migrated to V3 — plan-steps replacement isn't wired up for the V3 test-case schema yet. This is a clean, expected denial (not a bug); there is currently no CLI-side workaround. + #### `testsprite project create` / `project update` Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Projects have **no description field** — `--description` is rejected client-side with a validation error (descriptions live on tests: `test create --description`). `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`. @@ -455,7 +513,7 @@ Batch `--report` flags apply only to `test run --all --wait` (and batch `test re #### `testsprite test rerun [test-id...]` -Re-execute one or more tests as a cheap **replay** — distinct from `test run`, which triggers a fresh agent run that may regenerate code and spend credits. A frontend rerun replays the saved script (verbatim unless AI heal-on-drift engages — see `--auto-heal`); a backend rerun re-runs the named test together with its producer/teardown dependency closure. Without `--wait`, prints the queued run(s) and exits 0; with `--wait`, polls to terminal with the same exit-code matrix as `test run --wait`. +Re-execute one or more tests as a **replay** — distinct from `test run`, which triggers a fresh agent run that may regenerate code. A frontend rerun replays the saved script (verbatim unless AI heal-on-drift engages — see `--auto-heal`); a backend rerun re-runs the named test together with its producer/teardown dependency closure. A rerun is billed the same as a fresh run — 0.5 credits per FE rerun, 0.2 credits per BE rerun (legacy V2 accounts: FE rerun remains free). Without `--wait`, prints the queued run(s) and exits 0; with `--wait`, polls to terminal with the same exit-code matrix as `test run --wait`. ```bash # Frontend test — verbatim replay @@ -488,7 +546,7 @@ Flags: - `--all` — rerun every test in the resolved project; requires `--project `. - `--wait`, `--timeout ` — block until terminal; same exit matrix as `test run --wait`. -- `--auto-heal` / `--no-auto-heal` — frontend AI heal-on-drift, **on by default** for FE reruns; opt out with `--no-auto-heal`. Verbatim-replay passes are free; a heal engage costs a small amount of credit. Ignored for backend tests. +- `--auto-heal` / `--no-auto-heal` — frontend AI heal-on-drift, **on by default** for FE reruns; opt out with `--no-auto-heal`. The rerun itself is billed at 0.5 credits regardless of whether heal engages; a heal engage costs a small amount of credit on top of that (legacy V2 accounts: a verbatim-replay pass is free, and only a heal engage costs credit). Ignored for backend tests. On V3-routed accounts the `--no-auto-heal` opt-out is still rolling out and may not yet be honored server-side. - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. - `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). @@ -498,7 +556,7 @@ A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `defer #### `testsprite test flaky ` -Detect a **flaky** test by replaying it several times and reporting how often it passes. Each attempt is a rerun with auto-heal **off** (a strict verbatim replay), so healed drift can't disguise a nondeterministic pass/fail — this measures the replay stability of the saved script against the configured URL. Frontend replays are free verbatim script replays; backend tests re-run their dependency closure and may cost credits (a one-line stderr advisory is printed before the run). +Detect a **flaky** test by replaying it several times and reporting how often it passes. Each attempt is a rerun with auto-heal **off** (a strict verbatim replay), so healed drift can't disguise a nondeterministic pass/fail — this measures the replay stability of the saved script against the configured URL. Each replay is billed as a rerun, same as a fresh run: 0.5 credits for a frontend replay, 0.2 credits for a backend replay (legacy V2 accounts: FE rerun remains free) — so `--runs N` costs roughly N×0.5 credits for a frontend test. A one-line stderr advisory is printed before a backend replay. ```bash # Replay 10 times and print a stability score @@ -531,6 +589,12 @@ testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +A `RATE_LIMITED` (429) poll is the one per-member error that is retried before it becomes an outcome: each member re-polls up to 3 times, sleeping the server's `Retry-After`. Each backoff is clamped to the shared `--timeout` deadline, and a backoff interrupted by Ctrl-C detaches normally; if the deadline is reached during one, that member reports a **timeout** (exit 7), not a rate limit. + +If the throttle outlasts the retry budget **and** nothing else went wrong — no timeouts, no failed runs, no other error codes, and no repeated run id in the argument list — the exit code is **11** (rate limited) rather than 7, because the correct next action is to back off before re-attaching, not to retry immediately. Any timeout or non-passed run in the same invocation keeps the usual 7 / 1. + +One caveat this does not fix: the HTTP layer's own 429 retries (up to 3, honoring `Retry-After`) are bounded by their own budget, not by `--timeout`, so a sustained throttle can still overshoot the deadline by roughly one retry chain before the command gives up. That is pre-existing behavior on every polling command, not something this retry loop introduced — the outer loop re-checks the deadline before each of its own attempts. + #### `testsprite test cancel ` Cancel one or more in-flight runs — the counterpart to Ctrl-C, which only **detaches** (the server-side run keeps executing and billing). Cancelling is idempotent: an already-cancelled run reports `alreadyCancelled` as an advisory, not an error; a run that already reached a terminal verdict is a conflict — the verdict is never overwritten, and no credits are refunded. With one id, prints the run card; with several, prints a `{ cancelled, alreadyCancelled, conflicts, notFound }` summary. Exit codes: any unknown id → 4; else any conflict → 6; else 0. @@ -558,7 +622,7 @@ Returns 404 (CLI exit 4) when the run passed (`details.reason: "no_failing_run"` #### `testsprite usage` (alias: `testsprite credits`) -Account pre-flight before a large batch: resolves the active key to its identity (`userId`, `keyId`, `env`) and surfaces the credit balance / plan fields when the backend supplies them. Useful right before a `test run --all` fan-out. +Account pre-flight before a large batch: resolves the active key to its identity (`userId`, `keyId`, `env`) and surfaces the credit balance / plan fields when the backend supplies them. Useful right before a `test run --all` fan-out. For an org-scoped key, also prints the `orgs:` / `org binding:` lines described under [Authenticate](#1-authenticate). ```bash testsprite usage --output json @@ -576,7 +640,7 @@ testsprite doctor --output json testsprite doctor && testsprite test run test_xxxxxxxx --wait ``` -Every check reuses the same helpers the real commands use, so the report reflects exactly what a subsequent command would resolve. +Every check reuses the same helpers the real commands use, so the report reflects exactly what a subsequent command would resolve. For an org-scoped key, the report also lists `Organizations` (account-wide membership list) and `Org binding` (this key's bound organization) checks. ## Configuration @@ -600,18 +664,37 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ----------------------------------------- | ------------------------------------------------------------------------------------------------ | -| `TESTSPRITE_API_KEY` | API key - overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint - overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_PROJECT_ID` | Default project for `test list`, `test create`, and `test run --all` when `--project` is omitted | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`-`600000`) | -| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | -| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | -| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support - API traffic is routed through the configured proxy | -| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | -| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | +| Variable | Purpose | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------ | +| `TESTSPRITE_API_KEY` | API key - overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint - overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_PROJECT_ID` | Default project for `test list`, `test create`, and `test run --all` when `--project` is omitted | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`-`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support - API traffic is routed through the configured proxy | +| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | +| `TESTSPRITE_NO_TELEMETRY` / `DO_NOT_TRACK` | Any truthy value (not `0`/`false`/empty) disables usage telemetry (see Telemetry below) | +| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | + +### Telemetry + +Authenticated runs send one best-effort "command outcome" event per invocation +to TestSprite (`POST /api/cli/v1/telemetry`) so we can measure which commands +run and diagnose failures. Each event carries only: the command name (e.g. +`test run`), the outcome (`success`/`error`/`abort`), the exit code, a machine +error **code** (e.g. `VALIDATION_ERROR`), the duration, and context (CLI +version, OS, Node version, output mode, CI-vs-interactive). + +It **never** sends: your API key, target URLs, flag or argument values, or error +**messages**. The event is a fixed allowlist, bounded to ~1s, and fully +best-effort — it never delays beyond that, never changes a command's behavior or +exit code, and is skipped entirely when no API key is configured or under +`--dry-run`. + +Opt out with `TESTSPRITE_NO_TELEMETRY=1` or the cross-tool +`DO_NOT_TRACK=1` (any truthy value; `0`/`false`/empty do not opt out). ### Update notice @@ -668,7 +751,7 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not | `3` | Auth error | | `4` | Not found | | `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | +| `6` | Conflict / precondition failed / ambiguous org (see below) | | `7` | Timeout / unsupported | | `10` | Service unavailable | | `11` | Rate limited (retriable) | @@ -677,6 +760,15 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not | `14` | Client too old — the backend requires a newer CLI (HTTP 426 `CLIENT_TOO_OLD`); upgrade to proceed | | `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | +### Ambiguous org id (exit 6) + +For a membership-scoped API key, a testId can — pathologically — resolve to +projects in more than one of your organizations. The CLI prints one +`candidate: project (org )` line per colliding project plus a hint +to re-run with `--project `, and exits `6` (same family as a generic +conflict; retrying does not resolve it). `--output json` carries the same +information in `error.details.candidates`. + ### Signals & pipes During any `--wait`, SIGINT (Ctrl-C), SIGTERM, or SIGHUP triggers a **graceful detach**: the in-flight request aborts immediately, stdout gets the same partial `{ runId, status: "running" }` envelope as the request-timeout path (under `--output json`, stderr carries an `INTERRUPTED` envelope naming the signal), and stderr states the truth — the server-side run keeps executing, and any credit spend continues — with a re-attach hint (`test wait `) and a `test cancel ` pointer. The exit code is `128 + signal` (130 / 143 / 129). A second signal forces an immediate hard exit. Outside a `--wait` (prompts, one-shot commands), signals keep the pre-existing immediate-exit behavior. **Ctrl-C never cancels the server-side run** — `test cancel ` is the explicit stop. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. diff --git a/README.md b/README.md index 3bb465e..23945b1 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,31 @@ testsprite test rerun test_3a9f21c7 --wait --output json # → exits 0: passed. The test now lives in your durable suite. ``` +`./checkout-flow.plan.json` (the `--plan-from` argument above) is a JSON file describing the test in plain language — no browser code required. This is the exact, byte-identical output of `testsprite test create --plan-template` (also embedded verbatim in `test create --help`): + +```json +{ + "$schema": "https://raw.githubusercontent.com/TestSprite/testsprite-cli/v0.4.0/schemas/plan.schema.json", + "projectId": "prj_abc123", + "type": "frontend", + "name": "Login rejects an empty password", + "planSteps": [ + { + "type": "action", + "description": "Navigate to /login and submit the form with an empty password" + }, + { + "type": "assertion", + "description": "Verify an inline error says the password is required" + } + ] +} +``` + +Get this exact skeleton without hand-copying it (and without the risk of it drifting from your installed version — see below): `testsprite test create --plan-template`. Full field reference (including the `{{...}}`-placeholder caveat, size caps, and the `$schema` hook for live editor validation): [Plan file format](./DOCUMENTATION.md#plan-file-format). + +> The `$schema` URL above is pinned to the CLI version that generated this doc (`v0.4.0`) — `--plan-template`'s live output always pins to **your installed version** instead, which is what actually resolves. If you're reading this on a later release, run the command yourself rather than trusting this snippet verbatim. + Prefer to configure each step by hand (or learn the surface offline with `--dry-run` first)? See [Manual setup](./DOCUMENTATION.md#manual-setup) and [Install & verify](./DOCUMENTATION.md#install--verify). ## Commands @@ -114,7 +139,7 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- | | `project create` / `project update` / `project delete` | Manage projects; `delete` removes a project and everything under it (`--confirm` required, no restore window) | | | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | | **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test rerun` | Replay one/many tests (FE verbatim; BE with deps); billed as a run (0.5 credits FE / 0.2 BE); `--all --project ` reruns all tests | | | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | | | `test wait` | Block on one or more `runId`s until terminal | | | `test cancel` | Cancel one or more in-flight runs (Ctrl-C during `--wait` only detaches — `cancel` is the real stop) | diff --git a/package-lock.json b/package-lock.json index 4461016..085287a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.5.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", @@ -17,12 +17,13 @@ "testsprite": "dist/index.js" }, "devDependencies": { - "@eslint/js": "^9.14.0", + "@eslint/js": "^10.0.1", "@types/node": "^22.9.0", "@vitest/coverage-v8": "^2.1.4", - "eslint": "^9.14.0", + "ajv": "^8.20.0", + "eslint": "^10.7.0", "eslint-config-prettier": "^9.1.0", - "globals": "^15.12.0", + "globals": "^17.7.0", "msw": "^2.14.3", "prettier": "^3.3.3", "typescript": "^5.6.3", @@ -538,118 +539,89 @@ } }, "node_modules/@eslint/config-array": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", - "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/object-schema": "^2.1.7", + "@eslint/object-schema": "^3.0.5", "debug": "^4.3.1", - "minimatch": "^3.1.5" + "minimatch": "^10.2.4" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/config-helpers": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", - "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.6.0.tgz", + "integrity": "sha512-ii6Bw9jJ2zi2cWA2Z+9/QZ/+3DX6kwaV5Q986D/CdP3Lap3w/pgQZ373FV7byY/i7L4IRH/G43I5dz1ClsCbpA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.17.0" + "@eslint/core": "^1.2.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/core": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", - "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", "dev": true, "license": "Apache-2.0", "dependencies": { "@types/json-schema": "^7.0.15" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/eslintrc": { - "version": "3.3.5", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", - "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^6.14.0", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.1", - "minimatch": "^3.1.5", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint/eslintrc/node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/js": { - "version": "9.39.4", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", - "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", "dev": true, "license": "MIT", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } } }, "node_modules/@eslint/object-schema": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", - "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@eslint/plugin-kit": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", - "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.2.tgz", + "integrity": "sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@eslint/core": "^0.17.0", + "@eslint/core": "^1.2.1", "levn": "^0.4.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@humanfs/core": { @@ -1283,6 +1255,13 @@ "win32" ] }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -1325,17 +1304,17 @@ "license": "MIT" }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.59.1.tgz", - "integrity": "sha512-BOziFIfE+6osHO9FoJG4zjoHUcvI7fTNBSpdAwrNH0/TLvzjsk2oo8XSSOT2HhqUyhZPfHv4UOffoJ9oEEQ7Ag==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.64.0.tgz", + "integrity": "sha512-CGvQPBxN3wZLu6Rz2kFUpZeoCm78xUic92ck39KPePkO1NPOwjCqdQnm5Q87tpWw9vcBvW8XLrDXjH9PWYtJ3Q==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.59.1", - "@typescript-eslint/type-utils": "8.59.1", - "@typescript-eslint/utils": "8.59.1", - "@typescript-eslint/visitor-keys": "8.59.1", + "@typescript-eslint/scope-manager": "8.64.0", + "@typescript-eslint/type-utils": "8.64.0", + "@typescript-eslint/utils": "8.64.0", + "@typescript-eslint/visitor-keys": "8.64.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" @@ -1348,15 +1327,15 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.59.1", + "@typescript-eslint/parser": "^8.64.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", + "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", "dev": true, "license": "MIT", "engines": { @@ -1364,16 +1343,16 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.59.1.tgz", - "integrity": "sha512-HDQH9O/47Dxi1ceDhBXdaldtf/WV9yRYMjbjCuNk3qnaTD564qwv61Y7+gTxwxRKzSrgO5uhtw584igXVuuZkA==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.64.0.tgz", + "integrity": "sha512-KA0OshtlcCCXmbfqyZkM5pV3/WNraJf7DkJRLpyrmwPtud57H5BDX7C3k0LPSPxpprfRL+cJDGabF10mvNCoCw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/scope-manager": "8.59.1", - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/typescript-estree": "8.59.1", - "@typescript-eslint/visitor-keys": "8.59.1", + "@typescript-eslint/scope-manager": "8.64.0", + "@typescript-eslint/types": "8.64.0", + "@typescript-eslint/typescript-estree": "8.64.0", + "@typescript-eslint/visitor-keys": "8.64.0", "debug": "^4.4.3" }, "engines": { @@ -1389,14 +1368,14 @@ } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.59.1.tgz", - "integrity": "sha512-+MuHQlHiEr00Of/IQbE/MmEoi44znZHbR/Pz7Opq4HryUOlRi+/44dro9Ycy8Fyo+/024IWtw8m4JUMCGTYxDg==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.64.0.tgz", + "integrity": "sha512-tk4WpOJ6IEbGrVHaNmM0YRrwAD3exZlIK3iadQNAxh4YKk6jvUQ4ecq18n+v7+meh+cJ3j+D8nbk8sRKhlwLQg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.59.1", - "@typescript-eslint/types": "^8.59.1", + "@typescript-eslint/tsconfig-utils": "^8.64.0", + "@typescript-eslint/types": "^8.64.0", "debug": "^4.4.3" }, "engines": { @@ -1411,14 +1390,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.59.1.tgz", - "integrity": "sha512-LwuHQI4pDOYVKvmH2dkaJo6YZCSgouVgnS/z7yBPKBMvgtBvyLqiLy9Z6b7+m/TRcX1NFYUqZetI5Y+aT4GEfg==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.64.0.tgz", + "integrity": "sha512-CXEaFdYXjSTgKhisNkwCcJwTP8Pl+fmRrEQrri4nm3vU743bALrxzLmq7fHG/7e6a5xO0lDYeURpZmBuhHk54w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/visitor-keys": "8.59.1" + "@typescript-eslint/types": "8.64.0", + "@typescript-eslint/visitor-keys": "8.64.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1429,9 +1408,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.59.1.tgz", - "integrity": "sha512-/0nEyPbX7gRsk0Uwfe4ALwwgxuA66d/l2mhRDNlAvaj4U3juhUtJNq0DsY8M2AYwwb9rEq2hrC3IcIcEt++iJA==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.64.0.tgz", + "integrity": "sha512-2yo8rRNKuzbVWQp5kslhANqZ2uDAeROQHBRZNPu8JDsHmeFNj/XJJhX/FhNUWmkHHvoNsKa6+tHJiig87EzsQw==", "dev": true, "license": "MIT", "engines": { @@ -1446,15 +1425,15 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.59.1.tgz", - "integrity": "sha512-klWPBR2ciQHS3f++ug/mVnWKPjBUo7icEL3FAO1lhAR1Z1i5NQYZ1EannMSRYcq5qCv5wNALlXr6fksRHyYl7w==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.64.0.tgz", + "integrity": "sha512-XWG4Fmmv/6SvyS9nH8jWrKs6terwJvE8cyRt1CzYYqzp9OrPhCT4cMc/f7C6RZCwG+qMmiffJS1/qJP8G1URtg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/typescript-estree": "8.59.1", - "@typescript-eslint/utils": "8.59.1", + "@typescript-eslint/types": "8.64.0", + "@typescript-eslint/typescript-estree": "8.64.0", + "@typescript-eslint/utils": "8.64.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, @@ -1471,9 +1450,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.59.1.tgz", - "integrity": "sha512-ZDCjgccSdYPw5Bxh+my4Z0lJU96ZDN7jbBzvmEn0FZx3RtU1C7VWl6NbDx94bwY3V5YsgwRzJPOgeY2Q/nLG8A==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.64.0.tgz", + "integrity": "sha512-qjhfuTfLXjA4IOzXvz0rTjT01BqEiIgPoUeMwiEjnaHKJMTNo8rH5pYW1a2L/0Dnux2fPC85AeyJoWaGa8WxTA==", "dev": true, "license": "MIT", "engines": { @@ -1485,16 +1464,16 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.59.1.tgz", - "integrity": "sha512-OUd+vJS05sSkOip+BkZ/2NS8RMxrAAJemsC6vU3kmfLyeaJT0TftHkV9mcx2107MmsBVXXexhVu4F0TZXyMl4g==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.64.0.tgz", + "integrity": "sha512-Pztpsn1aCE1oWDvDEfUk31nngvvF7vUB5SwHFEaZIFpvw7WJtqUHHL4plBZDA9HfWJJjL13BdG0YrJInTUvoVA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.59.1", - "@typescript-eslint/tsconfig-utils": "8.59.1", - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/visitor-keys": "8.59.1", + "@typescript-eslint/project-service": "8.64.0", + "@typescript-eslint/tsconfig-utils": "8.64.0", + "@typescript-eslint/types": "8.64.0", + "@typescript-eslint/visitor-keys": "8.64.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", @@ -1512,56 +1491,17 @@ "typescript": ">=4.8.4 <6.1.0" } }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/@typescript-eslint/utils": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.59.1.tgz", - "integrity": "sha512-3pIeoXhCeYH9FSCBI8P3iNwJlGuzPlYKkTlen2O9T1DSeeg8UG8jstq6BLk+Mda0qup7mgk4z4XL4OzRaxZ8LA==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.64.0.tgz", + "integrity": "sha512-aJUGVB3+U0htrrCjoA8qukw8cm8fNCGAxK/tVoS70k8aeb7DETKeFozRiVFIwEeN9WJLsjaP3ph8I60tY2XZoQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.59.1", - "@typescript-eslint/types": "8.59.1", - "@typescript-eslint/typescript-estree": "8.59.1" + "@typescript-eslint/scope-manager": "8.64.0", + "@typescript-eslint/types": "8.64.0", + "@typescript-eslint/typescript-estree": "8.64.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1576,13 +1516,13 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.59.1.tgz", - "integrity": "sha512-LdDNl6C5iJExcM0Yh0PwAIBb9PrSiCsWamF/JyEZawm3kFDnRoaq3LGE4bpyRao/fWeGKKyw7icx0YxrLFC5Cg==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.64.0.tgz", + "integrity": "sha512-mrtuL8Nsn6gi2H4mo5KMTp823M+3Q19Ew/i+Zlikq20tIMm99C3Ez0dCmkWWnxut20esQvTg8aUSEhMcAOXhEw==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.59.1", + "@typescript-eslint/types": "8.64.0", "eslint-visitor-keys": "^5.0.0" }, "engines": { @@ -1593,19 +1533,6 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, "node_modules/@vitest/coverage-v8": { "version": "2.1.9", "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-2.1.9.tgz", @@ -1753,9 +1680,9 @@ } }, "node_modules/acorn": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", - "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", + "version": "8.17.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.17.0.tgz", + "integrity": "sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==", "dev": true, "license": "MIT", "bin": { @@ -1776,16 +1703,16 @@ } }, "node_modules/ajv": { - "version": "6.15.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", - "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", "dev": true, "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" }, "funding": { "type": "github", @@ -1821,13 +1748,6 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -1846,34 +1766,36 @@ "license": "MIT" }, "node_modules/brace-expansion": { - "version": "1.1.14", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.14.tgz", - "integrity": "sha512-MWPGfDxnyzKU7rNOW9SP/c50vi3xrmrua/+6hfPbCS2ABNWfx24vPidzvC7krjU/RTo235sV776ymlsMtGKj8g==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.7.tgz", + "integrity": "sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==", "dev": true, "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" } }, - "node_modules/cac": { - "version": "6.7.14", - "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", - "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", + "node_modules/brace-expansion/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "dev": true, "license": "MIT", "engines": { - "node": ">=8" + "node": "18 || 20 || >=22" } }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "node_modules/cac": { + "version": "6.7.14", + "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", + "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", "dev": true, "license": "MIT", "engines": { - "node": ">=6" + "node": ">=8" } }, "node_modules/chai": { @@ -1893,23 +1815,6 @@ "node": ">=18" } }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, "node_modules/check-error": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", @@ -2037,13 +1942,6 @@ "node": ">=18" } }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, "node_modules/cookie": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", @@ -2192,33 +2090,33 @@ } }, "node_modules/eslint": { - "version": "9.39.4", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", - "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", + "version": "10.7.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.7.0.tgz", + "integrity": "sha512-GVTD7s1vdIl6UYvAfriOPeY1Df8LIZjfofLvHwde+erDHGGuHyuM6xoxRxmHiebhYuD2p1vN4wWh0XzPARSGDQ==", "dev": true, "license": "MIT", + "workspaces": [ + "packages/*" + ], "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", - "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.2", - "@eslint/config-helpers": "^0.4.2", - "@eslint/core": "^0.17.0", - "@eslint/eslintrc": "^3.3.5", - "@eslint/js": "9.39.4", - "@eslint/plugin-kit": "^0.4.1", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.6.0", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.2", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.4.2", "@types/estree": "^1.0.6", "ajv": "^6.14.0", - "chalk": "^4.0.0", "cross-spawn": "^7.0.6", "debug": "^4.3.2", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.4.0", - "eslint-visitor-keys": "^4.2.1", - "espree": "^10.4.0", - "esquery": "^1.5.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^8.0.0", @@ -2228,8 +2126,7 @@ "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", "json-stable-stringify-without-jsonify": "^1.0.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.5", + "minimatch": "^10.2.4", "natural-compare": "^1.4.0", "optionator": "^0.9.3" }, @@ -2237,7 +2134,7 @@ "eslint": "bin/eslint.js" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://eslint.org/donate" @@ -2265,48 +2162,74 @@ } }, "node_modules/eslint-scope": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", - "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", "dev": true, "license": "BSD-2-Clause", "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", "esrecurse": "^4.3.0", "estraverse": "^5.2.0" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", "dev": true, "license": "Apache-2.0", "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/eslint/node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, "node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", "dev": true, "license": "BSD-2-Clause", "dependencies": { - "acorn": "^8.15.0", + "acorn": "^8.16.0", "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" + "eslint-visitor-keys": "^5.0.1" }, "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "funding": { "url": "https://opencollective.com/eslint" @@ -2416,6 +2339,23 @@ "fast-string-truncated-width": "^3.0.2" } }, + "node_modules/fast-uri": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.3.tgz", + "integrity": "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fast-wrap-ansi": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.0.tgz", @@ -2599,9 +2539,9 @@ } }, "node_modules/globals": { - "version": "15.15.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-15.15.0.tgz", - "integrity": "sha512-7ACyT3wmyp3I61S4fG682L0VA2RGD9otkqGJIwNUMF1SWUombIIk+af1unuDYgMm082aHYwD+mzJvv9Iu8dsgg==", + "version": "17.7.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.7.0.tgz", + "integrity": "sha512-Czmyns5dUsq4seFBR/Kdydhmo8y9kC79hiSkPn0YcGtNnYWnrgt0vjrSjx9tspoDGWm2CMarffRuLjM4xUz8xg==", "dev": true, "license": "MIT", "engines": { @@ -2659,23 +2599,6 @@ "node": ">= 4" } }, - "node_modules/import-fresh": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", - "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -2803,19 +2726,6 @@ "@pkgjs/parseargs": "^0.11.0" } }, - "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -2824,9 +2734,9 @@ "license": "MIT" }, "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", "dev": true, "license": "MIT" }, @@ -2877,13 +2787,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true, - "license": "MIT" - }, "node_modules/loupe": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", @@ -2937,16 +2840,19 @@ } }, "node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "version": "10.2.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", + "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^5.0.5" }, "engines": { - "node": "*" + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/minipass": { @@ -2967,9 +2873,9 @@ "license": "MIT" }, "node_modules/msw": { - "version": "2.14.3", - "resolved": "https://registry.npmjs.org/msw/-/msw-2.14.3.tgz", - "integrity": "sha512-kk8G5cocVlJ4wsKMGZegn2H6XLOEKjbA+nSJE2354e/SRp4mDicCHUYnMXpymzVcVDCs+GUAsmNqSn+yHv4T2A==", + "version": "2.15.0", + "resolved": "https://registry.npmjs.org/msw/-/msw-2.15.0.tgz", + "integrity": "sha512-2wQAmKkQKxRuXvYJxVhPGG0wZNBQyD06oJvxqw90XqLvptdqxdlHrFUfEteKkpaNORX3Xzc+HtEl/q0nfmN2wQ==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3111,19 +3017,6 @@ "dev": true, "license": "BlueOak-1.0.0" }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -3193,9 +3086,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", "engines": { @@ -3245,9 +3138,9 @@ } }, "node_modules/prettier": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.3.tgz", - "integrity": "sha512-7igPTM53cGHMW8xWuVTydi2KO233VFiTNyF5hLJqpilHfmn8C8gPf+PS7dUT64YcXFbiMGZxS9pCSxL/Dxm/Jw==", + "version": "3.9.5", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.5.tgz", + "integrity": "sha512-/FVl766LpUfB5vXgCYOYa0MeV/441Ia99AeICQIQFTY/Nw0roZwULcXpku5i1/m5kt/baz+s4Zogspd839HSMg==", "dev": true, "license": "MIT", "bin": { @@ -3280,14 +3173,14 @@ "node": ">=0.10.0" } }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", "dev": true, "license": "MIT", "engines": { - "node": ">=4" + "node": ">=0.10.0" } }, "node_modules/rettime": { @@ -3550,19 +3443,6 @@ "node": ">=8" } }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -3604,45 +3484,6 @@ "node": ">=18" } }, - "node_modules/test-exclude/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/test-exclude/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/test-exclude/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -3658,9 +3499,9 @@ "license": "MIT" }, "node_modules/tinyglobby": { - "version": "0.2.16", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", - "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { @@ -3794,16 +3635,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.59.1", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.59.1.tgz", - "integrity": "sha512-xqDcFVBmlrltH64lklOVp1wYxgJr6LVdg3NamBgH2OOQDLFdTKfIZXF5PfghrnXQKXZGTQs8tr1vL7fJvq8CTQ==", + "version": "8.64.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.64.0.tgz", + "integrity": "sha512-0qg+pDNMnqYzqH9AnNK+39tejHvsShUOUUoRUgtnTGE7QuMZhiFDnozq8nHJVq+Wae6NMLKNWLg5WmkcC/ndyQ==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.59.1", - "@typescript-eslint/parser": "8.59.1", - "@typescript-eslint/typescript-estree": "8.59.1", - "@typescript-eslint/utils": "8.59.1" + "@typescript-eslint/eslint-plugin": "8.64.0", + "@typescript-eslint/parser": "8.64.0", + "@typescript-eslint/typescript-estree": "8.64.0", + "@typescript-eslint/utils": "8.64.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3854,9 +3695,9 @@ } }, "node_modules/valibot": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/valibot/-/valibot-1.4.1.tgz", - "integrity": "sha512-klCmFTz2jeDluy9RwX+F884TCiogtdBJ/YaxSx1EOBYXa3NXNWj8kR1jjN8rzluwojJVWWaHJ4r1U5LfICnM3g==", + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/valibot/-/valibot-1.4.2.tgz", + "integrity": "sha512-gjdCvJ6d3RyHAneqxMYMW9QMCwYMb3jpOO0IyHZV1bnRHFBHrX3VkIILt5XYR0WhwHiH7Mty8ovuPZ/O3gamrg==", "license": "MIT", "peerDependencies": { "typescript": ">=5" diff --git a/package.json b/package.json index 18336cb..e88b123 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.4.0", + "version": "0.5.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", @@ -10,6 +10,7 @@ "files": [ "dist", "skills", + "schemas", "!dist/**/*.map" ], "scripts": { @@ -44,7 +45,16 @@ "testsprite", "cli", "testing", - "automation" + "automation", + "test-automation", + "ai-testing", + "e2e", + "qa", + "api-testing", + "playwright", + "mcp", + "claude-code", + "agent" ], "author": "TestSprite team", "license": "Apache-2.0", @@ -54,12 +64,13 @@ "valibot": "^1.4.1" }, "devDependencies": { - "@eslint/js": "^9.14.0", + "@eslint/js": "^10.0.1", "@types/node": "^22.9.0", "@vitest/coverage-v8": "^2.1.4", - "eslint": "^9.14.0", + "ajv": "^8.20.0", + "eslint": "^10.7.0", "eslint-config-prettier": "^9.1.0", - "globals": "^15.12.0", + "globals": "^17.7.0", "msw": "^2.14.3", "prettier": "^3.3.3", "typescript": "^5.6.3", diff --git a/schemas/plan.schema.json b/schemas/plan.schema.json new file mode 100644 index 0000000..788b6f1 --- /dev/null +++ b/schemas/plan.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://raw.githubusercontent.com/TestSprite/testsprite-cli/main/schemas/plan.schema.json", + "title": "TestSprite CLI plan file", + "description": "Shape ingested by `testsprite test create --plan-from ` (and, per-line, by `test create-batch --plans ` / `--plan-from-dir `). One plan file describes exactly ONE test — a top-level JSON array is rejected; use create-batch for many. This schema is the machine-readable ground truth for THIS command (`--plan-from`), not for `assertPlanShape`'s raw structural check in isolation — where the two diverge (see `type` below), this schema encodes what `--plan-from` actually accepts end-to-end. It is asserted in `src/lib/plan-schema.spec.ts` against the real validator; if the two ever disagree, the validator's actual acceptance behavior is authoritative and this file is out of date. Not enforced by this schema (checked separately by the CLI as a pre-flight guard, not a structural JSON constraint): a plan file must be <= 256 KB total.", + "type": "object", + "required": ["projectId", "type", "name", "planSteps"], + "properties": { + "$schema": { + "description": "Optional. Point your editor at this file (e.g. the raw URL in the plan template, or a local path to a copy shipped in node_modules/@testsprite/testsprite-cli/schemas/plan.schema.json) to get inline validation and completion while authoring. Ignored by the CLI — plan files are not restricted to a fixed property set (see `additionalProperties`, implicitly true throughout this schema), and this property is deliberately left type-unconstrained here to match: assertPlanShape does not inspect `$schema` at all, so a non-string value here is exactly as valid to the CLI as a string one." + }, + "projectId": { + "type": "string", + "minLength": 1, + "pattern": "\\S", + "description": "Project id returned by `testsprite project list`. Must be non-empty and not whitespace-only." + }, + "type": { + "type": "string", + "enum": ["frontend"], + "description": "Restricted to `frontend` — `--plan-from` rejects a `backend`-typed plan pre-flight with a `nextAction` pointing at `test create --type backend --code-file ` (backend tests are authored from a code file, not a plan). `assertPlanShape`'s own structural check is looser (it accepts `backend` as a bare shape check before that later rejection), but this schema is the ground truth for the `--plan-from` COMMAND as a whole, not for the isolated structural check — so it encodes the value that actually succeeds end-to-end." + }, + "name": { + "type": "string", + "minLength": 1, + "pattern": "\\S", + "description": "Human-readable test name. Write it as an assertable behavior statement (subject + verb + outcome), not a noun fragment." + }, + "description": { + "type": "string", + "description": "Optional one-sentence elaboration of `name` — the condition plus the expected outcome. Any string is accepted (including empty), unlike `name`/`projectId`." + }, + "priority": { + "type": "string", + "enum": ["p0", "p1", "p2", "p3"], + "description": "Optional. p0 = must-pass, p1 = important paths, p2 = edge cases, p3 = cosmetic." + }, + "planSteps": { + "type": "array", + "minItems": 1, + "maxItems": 200, + "description": "1-200 steps describing user intent in plain language, not selectors. `{{...}}`-style placeholders are NOT substituted by the CLI — a step containing one is structurally valid (passes this schema and assertPlanShape) but triggers a non-fatal `[advisory]` at `test create --plan-from` time; store login credentials on the project instead (`project update --username/--password`, or Portal -> Project Settings).", + "items": { + "type": "object", + "required": ["type", "description"], + "properties": { + "type": { + "type": "string", + "enum": ["action", "assertion"] + }, + "description": { + "type": "string", + "minLength": 1, + "pattern": "\\S", + "description": "One verb per step. Describe the outcome/intent, not a CSS selector or literal button label." + } + } + } + } + } +} diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 7326aeb..87456c8 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -925,6 +925,117 @@ describe('createAgentCommand wiring', () => { expect(out).toContain('claude'); expect(out).toContain('antigravity'); }); + + // ------------------------------------------------------------------------- + // `agent install ` positional argument + // ------------------------------------------------------------------------- + // + // Before the fix, `install` declared only `--target ` with no + // `.argument()`, so Commander silently dropped an excess positional and + // `install` fell through to the non-TTY default-to-claude path regardless + // of what was typed — `agent install cursor` installed claude's skill with + // exit 0 and zero signal. These tests exercise the real Commander wiring + // (not just `runInstall` directly) so a regression in the `.argument()` + // declaration itself would be caught, not just a regression in the + // downstream parsing logic. + + it('agent install (positional) via parseAsync installs the named target, not the claude default', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + const command = createAgentCommand({ cwd: CWD, fs: agentFs, ...deps }); + const parent = new (await import('commander')).Command('testsprite'); + parent.option('--output ', 'output', 'text'); + parent.option('--profile ', 'profile', 'default'); + parent.option('--endpoint-url '); + parent.option('--debug', 'debug', false); + parent.option('--verbose', 'verbose', false); + parent.option('--dry-run', 'dry-run', false); + parent.addCommand(command); + + await parent.parseAsync(['node', 'ts', 'agent', 'install', 'cursor', `--dir=${CWD}`]); + + const cursorAbs = path.resolve(CWD, pathFor('cursor', 'testsprite-verify')); + const claudeAbs = path.resolve(CWD, pathFor('claude', 'testsprite-verify')); + expect(store.get(cursorAbs)).toBe(renderForTarget('cursor', 'testsprite-verify').content); + expect(store.has(claudeAbs)).toBe(false); + }); + + it('agent install (multiple positionals) via parseAsync installs both', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + const command = createAgentCommand({ cwd: CWD, fs: agentFs, ...deps }); + const parent = new (await import('commander')).Command('testsprite'); + parent.option('--output ', 'output', 'text'); + parent.option('--profile ', 'profile', 'default'); + parent.option('--endpoint-url '); + parent.option('--debug', 'debug', false); + parent.option('--verbose', 'verbose', false); + parent.option('--dry-run', 'dry-run', false); + parent.addCommand(command); + + await parent.parseAsync(['node', 'ts', 'agent', 'install', 'cline', 'kiro', `--dir=${CWD}`]); + + expect(store.has(path.resolve(CWD, pathFor('cline', 'testsprite-verify')))).toBe(true); + expect(store.has(path.resolve(CWD, pathFor('kiro', 'testsprite-verify')))).toBe(true); + }); + + it('agent install --target via parseAsync merges positional and flag targets', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + const command = createAgentCommand({ cwd: CWD, fs: agentFs, ...deps }); + const parent = new (await import('commander')).Command('testsprite'); + parent.option('--output ', 'output', 'text'); + parent.option('--profile ', 'profile', 'default'); + parent.option('--endpoint-url '); + parent.option('--debug', 'debug', false); + parent.option('--verbose', 'verbose', false); + parent.option('--dry-run', 'dry-run', false); + parent.addCommand(command); + + await parent.parseAsync([ + 'node', + 'ts', + 'agent', + 'install', + 'antigravity', + '--target=windsurf', + `--dir=${CWD}`, + ]); + + expect(store.has(path.resolve(CWD, pathFor('antigravity', 'testsprite-verify')))).toBe(true); + expect(store.has(path.resolve(CWD, pathFor('windsurf', 'testsprite-verify')))).toBe(true); + }); + + it('agent install (positional) via parseAsync throws CLIError exit 5', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + + const command = createAgentCommand({ cwd: CWD, fs: agentFs, ...deps }); + const parent = new (await import('commander')).Command('testsprite'); + parent.option('--output ', 'output', 'text'); + parent.option('--profile ', 'profile', 'default'); + parent.option('--endpoint-url '); + parent.option('--debug', 'debug', false); + parent.option('--verbose', 'verbose', false); + parent.option('--dry-run', 'dry-run', false); + parent.addCommand(command); + + let thrown: unknown; + try { + await parent.parseAsync(['node', 'ts', 'agent', 'install', 'banana', `--dir=${CWD}`]); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeDefined(); + const isValidationErr = + (thrown instanceof ApiError && thrown.exitCode === 5) || + (thrown instanceof CLIError && thrown.exitCode === 5); + expect(isValidationErr).toBe(true); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 7d7e7c8..2e5fa9e 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -524,7 +524,7 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr const bytes = Buffer.byteLength(section, 'utf8'); let wouldBeContent = section; if (dryRunSt !== null) { - let existing: string | null = null; + let existing: string | null; try { existing = await agentFs.readFile(abs); } catch (err) { @@ -1066,13 +1066,14 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ); agent - .command('install') + .command('install [targets...]') .description( - 'Write the TestSprite agent skills (verification loop + first-run onboarding) into a project for a coding agent', + 'Write the TestSprite agent skills (verification loop + first-run onboarding) into a project for a coding agent. ' + + 'Target(s) may be given positionally (e.g. `agent install cursor codex`) and/or via --target; the two are merged.', ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, copilot, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, copilot, codex (comma-separated or repeated). Merged with any positional target(s).', collect, [], ) @@ -1091,13 +1092,23 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { .addHelpText('after', GLOBAL_OPTS_HINT) .action( async ( + // Positional targets: `agent install cursor` previously parsed + // as zero targets (Commander silently drops undeclared positionals), + // silently falling through to the non-TTY default-to-claude path — so + // 7 of the 8 documented one-liners installed the WRONG agent's skill + // with zero signal. Declaring `[targets...]` captures them; they are + // merged with `--target` (order: positional first, then flag values) + // and flow through runInstall's existing parse/validate/dedupe pipeline + // unchanged, so an unknown name (positional or flag) still rejects with + // exit 5 instead of silently defaulting. + positionalTargets: string[], cmdOpts: { target: string[]; skill: string[]; dir?: string; force?: boolean }, command: Command, ) => { await runInstall( { ...resolveCommonOptions(command), - target: cmdOpts.target, + target: [...positionalTargets, ...cmdOpts.target], skills: cmdOpts.skill, dir: cmdOpts.dir, force: Boolean(cmdOpts.force), diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 0ce0bfb..82ca013 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -57,25 +57,42 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-from-env', TESTSPRITE_API_URL: 'https://from-env' }, + env: { TESTSPRITE_API_KEY: 'sk-user-from-env', TESTSPRITE_API_URL: 'https://from-env' }, credentialsPath, fetchImpl: meOkFetch, }, ); expect(readProfile('default', { path: credentialsPath })).toEqual({ - apiKey: 'sk-from-env', + apiKey: 'sk-user-from-env', apiUrl: 'https://from-env', }); expect(capture.stdout.join('\n')).toContain('configured'); }); + it('rejects a malformed API key up front, before the pre-write /me ping', async () => { + const { deps } = makeCapture(); + const fetchImpl = vi.fn(); + await expect( + runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'not-a-valid-key-format!!' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('--from-env without TESTSPRITE_API_URL uses the built-in default endpoint', async () => { const { deps } = makeCapture(); await runConfigure( { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk' }, + env: { TESTSPRITE_API_KEY: 'sk-user-min' }, credentialsPath, fetchImpl: meOkFetch, }, @@ -98,7 +115,7 @@ describe('runConfigure', () => { { ...deps, env: { - TESTSPRITE_API_KEY: 'sk', + TESTSPRITE_API_KEY: 'sk-user-min', TESTSPRITE_API_URL: 'https://env-loses.example.com', }, credentialsPath, @@ -143,7 +160,7 @@ describe('runConfigure', () => { }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk' }, + env: { TESTSPRITE_API_KEY: 'sk-user-min' }, credentialsPath, fetchImpl, }, @@ -179,7 +196,7 @@ describe('runConfigure', () => { }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk' }, + env: { TESTSPRITE_API_KEY: 'sk-user-min' }, credentialsPath, fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], }, @@ -210,7 +227,7 @@ describe('runConfigure', () => { }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk' }, + env: { TESTSPRITE_API_KEY: 'sk-user-min' }, credentialsPath, fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], }, @@ -264,7 +281,7 @@ describe('runConfigure', () => { // the endpoint it would call an undefined `text` and throw — so a passing // test proves the endpoint is never prompted. const prompt = { - secret: vi.fn(async () => 'sk-typed'), + secret: vi.fn(async () => 'sk-user-typed'), }; await runConfigure( { profile: 'default', output: 'text', debug: false, fromEnv: false }, @@ -272,7 +289,7 @@ describe('runConfigure', () => { ); expect(prompt.secret).toHaveBeenCalledTimes(1); expect(readProfile('default', { path: credentialsPath })).toEqual({ - apiKey: 'sk-typed', + apiKey: 'sk-user-typed', apiUrl: 'https://api.testsprite.com', }); expect(capture.prelude.join('')).toContain('Configuring profile "default"'); @@ -294,7 +311,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: false }, { stdout: line => stdout.push(line), - prompt: { secret: vi.fn(async () => 'sk-typed') }, + prompt: { secret: vi.fn(async () => 'sk-user-typed') }, fetchImpl: meOkFetch, credentialsPath, env: {}, @@ -309,7 +326,7 @@ describe('runConfigure', () => { it('interactive path resolves the endpoint from TESTSPRITE_API_URL without prompting', async () => { const { capture, deps } = makeCapture(); - const prompt = { secret: vi.fn(async () => 'sk-typed') }; + const prompt = { secret: vi.fn(async () => 'sk-user-typed') }; await runConfigure( { profile: 'default', output: 'text', debug: false, fromEnv: false }, { @@ -321,7 +338,7 @@ describe('runConfigure', () => { }, ); expect(readProfile('default', { path: credentialsPath })).toEqual({ - apiKey: 'sk-typed', + apiKey: 'sk-user-typed', apiUrl: 'https://api.example.com:8443', }); // An env-supplied endpoint is explicit → no inherit advisory. @@ -334,16 +351,16 @@ describe('runConfigure', () => { // (the internal dogfooding flow) without ever prompting for the endpoint. writeProfile( 'dev', - { apiKey: 'sk-old', apiUrl: 'https://api.example.com:8443' }, + { apiKey: 'sk-user-old', apiUrl: 'https://api.example.com:8443' }, { path: credentialsPath }, ); - const prompt = { secret: vi.fn(async () => 'sk-typed') }; + const prompt = { secret: vi.fn(async () => 'sk-user-typed') }; await runConfigure( { profile: 'dev', output: 'text', debug: false, fromEnv: false }, { ...deps, env: {}, credentialsPath, prompt, fetchImpl: meOkFetch }, ); expect(readProfile('dev', { path: credentialsPath })).toEqual({ - apiKey: 'sk-typed', + apiKey: 'sk-user-typed', apiUrl: 'https://api.example.com:8443', }); expect(capture.stderr.join('\n')).toContain( @@ -364,7 +381,7 @@ describe('runConfigure', () => { it('honors --endpoint-url without prompting for the endpoint', async () => { const { capture, deps } = makeCapture(); - const prompt = { secret: vi.fn(async () => 'sk-1') }; + const prompt = { secret: vi.fn(async () => 'sk-user-1') }; await runConfigure( { profile: 'default', @@ -390,12 +407,12 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-good' }, + env: { TESTSPRITE_API_KEY: 'sk-user-good' }, credentialsPath, fetchImpl: meOkFetch, }, ); - expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-good'); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-user-good'); expect(capture.stdout.join('\n')).toContain('configured'); }); @@ -422,7 +439,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-bad' }, + env: { TESTSPRITE_API_KEY: 'sk-user-bad' }, credentialsPath, fetchImpl: rejectedFetch, }, @@ -462,7 +479,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'json', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-bad' }, + env: { TESTSPRITE_API_KEY: 'sk-user-bad' }, credentialsPath, fetchImpl: rejectedFetch, }, @@ -484,7 +501,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-good' }, + env: { TESTSPRITE_API_KEY: 'sk-user-good' }, credentialsPath, fetchImpl: meOkFetch, }, @@ -498,7 +515,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'json', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-good' }, + env: { TESTSPRITE_API_KEY: 'sk-user-good' }, credentialsPath, fetchImpl: meOkFetch, }, @@ -512,7 +529,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true, dryRun: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-good' }, + env: { TESTSPRITE_API_KEY: 'sk-user-good' }, credentialsPath, fetchImpl: meOkFetch, }, @@ -543,7 +560,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-bad' }, + env: { TESTSPRITE_API_KEY: 'sk-user-bad' }, credentialsPath, fetchImpl: rejectedFetch, }, @@ -559,7 +576,7 @@ describe('runConfigure', () => { // Pre-write an existing profile with a custom (non-default) endpoint. writeProfile( 'default', - { apiKey: 'sk-old', apiUrl: 'https://api.example.com' }, + { apiKey: 'sk-user-old', apiUrl: 'https://api.example.com' }, { path: credentialsPath }, ); // codex-review P2 (2026-05-28): capture the URL the /me ping was made against @@ -581,14 +598,14 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-new' }, + env: { TESTSPRITE_API_KEY: 'sk-user-new' }, credentialsPath, fetchImpl: urlAwareFetch, }, ); // The new profile should reuse the inherited dev endpoint. expect(readProfile('default', { path: credentialsPath })).toEqual({ - apiKey: 'sk-new', + apiKey: 'sk-user-new', apiUrl: 'https://api.example.com', }); // The /me ping MUST have been issued against the inherited dev URL — this is @@ -607,7 +624,7 @@ describe('runConfigure', () => { const { capture, deps } = makeCapture(); writeProfile( 'default', - { apiKey: 'sk-old', apiUrl: 'https://api.example.com' }, + { apiKey: 'sk-user-old', apiUrl: 'https://api.example.com' }, { path: credentialsPath }, ); await runConfigure( @@ -620,7 +637,7 @@ describe('runConfigure', () => { }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-new' }, + env: { TESTSPRITE_API_KEY: 'sk-user-new' }, credentialsPath, fetchImpl: meOkFetch, }, @@ -637,14 +654,14 @@ describe('runConfigure', () => { // Existing profile has the default prod endpoint — no advisory needed. writeProfile( 'default', - { apiKey: 'sk-old', apiUrl: 'https://api.testsprite.com' }, + { apiKey: 'sk-user-old', apiUrl: 'https://api.testsprite.com' }, { path: credentialsPath }, ); await runConfigure( { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-new' }, + env: { TESTSPRITE_API_KEY: 'sk-user-new' }, credentialsPath, fetchImpl: meOkFetch, }, @@ -659,7 +676,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-new' }, + env: { TESTSPRITE_API_KEY: 'sk-user-new' }, credentialsPath, fetchImpl: meOkFetch, }, @@ -692,7 +709,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-bad' }, + env: { TESTSPRITE_API_KEY: 'sk-user-bad' }, credentialsPath, fetchImpl: rejectedFetch, }, @@ -713,7 +730,7 @@ describe('runConfigure', () => { // to the existing profile's api_url. writeProfile( 'default', - { apiKey: 'sk-old', apiUrl: 'https://api.example.com:8443' }, + { apiKey: 'sk-user-old', apiUrl: 'https://api.example.com:8443' }, { path: credentialsPath }, ); const seenFetchUrls: string[] = []; @@ -730,7 +747,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'text', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-new', TESTSPRITE_API_URL: ' ' }, + env: { TESTSPRITE_API_KEY: 'sk-user-new', TESTSPRITE_API_URL: ' ' }, credentialsPath, fetchImpl: urlAwareFetch, }, @@ -765,7 +782,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'json', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk' }, + env: { TESTSPRITE_API_KEY: 'sk-user-min' }, credentialsPath, fetchImpl: capturingFetch, commandTag: 'init', @@ -790,7 +807,7 @@ describe('runConfigure', () => { { profile: 'default', output: 'json', debug: false, fromEnv: true }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk' }, + env: { TESTSPRITE_API_KEY: 'sk-user-min' }, credentialsPath, fetchImpl: capturingFetch, }, @@ -819,14 +836,14 @@ describe('runWhoami', () => { it('calls GET /me using the configured profile and prints text output', async () => { writeProfile( 'default', - { apiKey: 'sk-stored', apiUrl: 'https://api.example.com' }, + { apiKey: 'sk-user-stored', apiUrl: 'https://api.example.com' }, { path: credentialsPath }, ); const { capture, deps } = makeCapture(); const fetchImpl = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { expect(input.toString()).toBe('https://api.example.com/api/cli/v1/me'); const headers = new Headers(init?.headers); - expect(headers.get('x-api-key')).toBe('sk-stored'); + expect(headers.get('x-api-key')).toBe('sk-user-stored'); expect(headers.get('authorization')).toBeNull(); return meResponse(); }); @@ -840,7 +857,7 @@ describe('runWhoami', () => { }); it('emits JSON when --output json', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runWhoami( { profile: 'default', output: 'json', debug: false }, @@ -851,7 +868,7 @@ describe('runWhoami', () => { }); it('renders routing: v3 and the gap advisory when v3Enabled is true', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const meV3 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: true }), { status: 200, @@ -863,11 +880,11 @@ describe('runWhoami', () => { ); expect(capture.stdout.join('\n')).toContain('routing: v3'); expect(capture.stderr.join('\n')).toContain('[advisory]'); - expect(capture.stderr.join('\n')).toContain('test cancel'); + expect(capture.stderr.join('\n')).toContain('--target-url'); }); it('renders routing: v2 and NO advisory when v3Enabled is false', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const meV2 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: false }), { status: 200, @@ -882,7 +899,7 @@ describe('runWhoami', () => { }); it('omits the routing line when the backend does not return v3Enabled', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runWhoami( { profile: 'default', output: 'text', debug: false }, @@ -891,8 +908,74 @@ describe('runWhoami', () => { expect(capture.stdout.join('\n')).not.toContain('routing:'); }); + it('renders the org line when activeOrg is present on a V3-routed caller', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meOrg = new Response( + JSON.stringify({ + ...sampleMe, + v3Enabled: true, + activeOrg: { + id: 'org-1', + name: 'Acme QA', + plan: 'Standard', + role: 'member', + remaining: 1650, + includedCredits: 1600, + seats: 3, + }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meOrg) }, + ); + expect(capture.stdout.join('\n')).toContain('org: Acme QA (Standard, member)'); + }); + + it('omits the org line when activeOrg is present but the caller is not V3-routed', async () => { + // Wallet selection follows the authoritative routing bit, never field + // presence alone — a V2-routed caller's commands charge the legacy + // wallet, so no org context may be shown even if a backend regression + // ships activeOrg to them again. + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meOrgV2 = new Response( + JSON.stringify({ + ...sampleMe, + v3Enabled: false, + activeOrg: { + id: 'org-1', + name: 'Acme QA', + plan: 'Standard', + role: 'member', + remaining: 1650, + includedCredits: 1600, + seats: 3, + }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meOrgV2) }, + ); + expect(capture.stdout.join('\n')).not.toContain('org: Acme QA'); + }); + + it('omits the org line when activeOrg is absent (older backends)', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ); + expect(capture.stdout.join('\n')).not.toContain('org:'); + }); + it('does not emit the advisory in JSON mode even when v3Enabled is true', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const meV3 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: true }), { status: 200, @@ -907,6 +990,114 @@ describe('runWhoami', () => { expect(parsed.v3Enabled).toBe(true); }); + it('renders an `orgs:` line when the backend returns a non-empty organizations[]', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meWithOrgs = new Response( + JSON.stringify({ + ...sampleMe, + organizations: [ + { id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }, + { id: 'org_2', name: "u-1's workspace", role: 'owner', isPersonal: true }, + ], + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meWithOrgs) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain( + 'orgs: Acme Corp (org_1, role: owner); ' + "u-1's workspace (org_2, personal, role: owner)", + ); + }); + + it('renders an `org binding:` line when the backend returns a membership-key org binding', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meWithBinding = new Response( + JSON.stringify({ + ...sampleMe, + org: { id: 'org_1', name: 'Acme Corp', role: 'member' }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meWithBinding) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('org binding: Acme Corp (org_1, role: member)'); + }); + + it('falls back to the org id in `org binding:` when name is null (resolution failed server-side)', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meWithBinding = new Response( + JSON.stringify({ + ...sampleMe, + org: { id: 'org_1', name: null, role: 'member' }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meWithBinding) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('org binding: org_1 (org_1, role: member)'); + }); + + it('omits `orgs:` and `org binding:` lines entirely when the backend does not return them (older backend)', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ); + const out = capture.stdout.join('\n'); + expect(out).not.toContain('orgs:'); + expect(out).not.toContain('org binding:'); + expect(out).not.toContain('undefined'); + }); + + it('omits the `orgs:` line when organizations is present but empty', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meEmptyOrgs = new Response(JSON.stringify({ ...sampleMe, organizations: [] }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meEmptyOrgs) }, + ); + expect(capture.stdout.join('\n')).not.toContain('orgs:'); + }); + + it('--output json: organizations[] and org pass through verbatim', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meFull = new Response( + JSON.stringify({ + ...sampleMe, + organizations: [{ id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }], + org: { id: 'org_1', name: 'Acme Corp', role: 'owner' }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meFull) }, + ); + const parsed = JSON.parse(capture.stdout.join('')) as MeResponse; + expect(parsed.organizations).toEqual([ + { id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }, + ]); + expect(parsed.org).toEqual({ id: 'org_1', name: 'Acme Corp', role: 'owner' }); + }); + it('dry-run: whitespace-only TESTSPRITE_API_URL falls through to prod default endpoint', async () => { const { capture, deps } = makeCapture(); await runWhoami( @@ -924,7 +1115,7 @@ describe('runWhoami', () => { it('L1788: text output includes the resolved endpoint URL', async () => { writeProfile( 'default', - { apiKey: 'sk-stored', apiUrl: 'https://api.example.com' }, + { apiKey: 'sk-user-stored', apiUrl: 'https://api.example.com' }, { path: credentialsPath }, ); const { capture, deps } = makeCapture(); @@ -952,7 +1143,7 @@ describe('runWhoami', () => { it('L1788: JSON output does NOT add endpoint (raw /me envelope is passed through)', async () => { writeProfile( 'default', - { apiKey: 'sk-stored', apiUrl: 'https://api.example.com' }, + { apiKey: 'sk-user-stored', apiUrl: 'https://api.example.com' }, { path: credentialsPath }, ); const { capture, deps } = makeCapture(); @@ -979,7 +1170,7 @@ describe('runWhoami', () => { }); it('L1866: renders email + name in text mode when the backend supplies them', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const meWithEmail = new Response( JSON.stringify({ ...sampleMe, email: 'alice@example.com', displayName: 'Alice' }), @@ -996,7 +1187,7 @@ describe('runWhoami', () => { }); it('L1866: omits email/name lines when the backend does not return them', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runWhoami( { profile: 'default', output: 'text', debug: false }, @@ -1009,7 +1200,7 @@ describe('runWhoami', () => { }); it('L1866: passes email through verbatim in JSON mode when present', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const meWithEmail = new Response(JSON.stringify({ ...sampleMe, email: 'alice@example.com' }), { status: 200, @@ -1039,7 +1230,7 @@ describe('runWhoami', () => { { profile: 'default', output: 'text', debug: false }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-env' }, + env: { TESTSPRITE_API_KEY: 'sk-user-env' }, credentialsPath, fetchImpl: makeFetch(meResponse()), }, @@ -1047,7 +1238,7 @@ describe('runWhoami', () => { }); it('emits debug events to stderr when debug is enabled', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runWhoami( { profile: 'default', output: 'json', debug: true }, @@ -1060,7 +1251,7 @@ describe('runWhoami', () => { }); it('forwards server AUTH_INVALID with exit code 3', async () => { - writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-bad' }, { path: credentialsPath }); const { deps } = makeCapture(); const errorBody = { error: { @@ -1090,7 +1281,7 @@ describe('runWhoami', () => { ...sampleMe, scopes: ['read:projects', 'read:tests'], }; - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const fetchImpl = makeFetch( new Response(JSON.stringify(readOnlyMe), { @@ -1113,7 +1304,7 @@ describe('runWhoami', () => { ...sampleMe, scopes: ['read:projects', 'read:tests', 'write:tests', 'run:tests'], }; - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const fetchImpl = makeFetch( new Response(JSON.stringify(fullMe), { @@ -1134,7 +1325,7 @@ describe('runWhoami', () => { ...sampleMe, scopes: ['read:projects', 'read:tests'], }; - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const fetchImpl = makeFetch( new Response(JSON.stringify(readOnlyMe), { @@ -1156,8 +1347,8 @@ describe('runWhoami', () => { describe('runLogout', () => { it('removes the profile and reports success', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); - writeProfile('dev', { apiKey: 'sk-dev' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + writeProfile('dev', { apiKey: 'sk-user-dev' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runLogout( { profile: 'default', output: 'text', debug: false }, @@ -1205,7 +1396,7 @@ describe('createAuthCommand wiring', () => { }); it('remove deletes the active profile and exits 0', async () => { - writeProfile('default', { apiKey: 'sk-remove' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-remove' }, { path: credentialsPath }); const { deps } = makeCapture(); const auth = createAuthCommand({ ...deps, credentialsPath }); auth.exitOverride(); @@ -1215,7 +1406,7 @@ describe('createAuthCommand wiring', () => { }); it('deprecated `whoami` alias emits a deprecation notice pointing at `auth status`', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const auth = createAuthCommand({ ...deps, @@ -1231,7 +1422,7 @@ describe('createAuthCommand wiring', () => { }); it('whoami uses injected fetch and exits 0', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { deps } = makeCapture(); const fetchImpl = vi.fn( async () => @@ -1248,7 +1439,7 @@ describe('createAuthCommand wiring', () => { }); it('L1802: `status` alias resolves to the whoami action', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { deps } = makeCapture(); const fetchImpl = vi.fn( async () => @@ -1265,7 +1456,7 @@ describe('createAuthCommand wiring', () => { }); it('logout removes the profile', async () => { - writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { deps } = makeCapture(); const auth = createAuthCommand({ ...deps, credentialsPath }); auth.exitOverride(); @@ -1291,7 +1482,7 @@ describe('runConfigure -- skipIfConfigured', () => { const { capture, deps } = makeCapture(); // Write a saved key first. writeProfile('default', { apiKey: 'sk-existing' }, { path: credentialsPath }); - const prompt = { secret: vi.fn(async () => 'sk-new') }; + const prompt = { secret: vi.fn(async () => 'sk-user-new') }; const fetchImpl = vi.fn(); await runConfigure( @@ -1332,7 +1523,7 @@ describe('runConfigure -- skipIfConfigured', () => { it('proceeds normally when no credentials exist and skipIfConfigured is true', async () => { const { deps } = makeCapture(); // No pre-existing profile -- skip has no effect, should fall through to prompt. - const prompt = { secret: vi.fn(async () => 'sk-new') }; + const prompt = { secret: vi.fn(async () => 'sk-user-new') }; await runConfigure( { profile: 'default', output: 'text', debug: false, fromEnv: false, skipIfConfigured: true }, @@ -1340,7 +1531,7 @@ describe('runConfigure -- skipIfConfigured', () => { ); expect(prompt.secret).toHaveBeenCalledTimes(1); - expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-new'); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-user-new'); }); it('ignores skipIfConfigured when --from-env is set', async () => { @@ -1358,13 +1549,13 @@ describe('runConfigure -- skipIfConfigured', () => { }, { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-from-env' }, + env: { TESTSPRITE_API_KEY: 'sk-user-from-env' }, credentialsPath, fetchImpl: meOkFetch, }, ); // The env key must overwrite the saved key. - expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-from-env'); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-user-from-env'); }); }); diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 7e60c05..3e98a91 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -1,5 +1,6 @@ import { Command } from 'commander'; import { + assertValidApiKey, assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, @@ -22,6 +23,8 @@ import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptSecret } from '../lib/prompt.js'; +import type { CliOrgBinding, CliOrgSummary } from '../lib/org-render.js'; +import { formatOrgBinding, formatOrgsSummary, formatPersonalScopeHint } from '../lib/org-render.js'; import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; export interface MeResponse { @@ -43,6 +46,40 @@ export interface MeResponse { * so it is only rendered when present. */ v3Enabled?: boolean; + /** + * Legacy per-user billing projections. Absent-safe; superseded by + * `activeOrg` when both are present. Declared so shared fixtures typed as + * `MeResponse` can carry them (the `usage` command has its own richer view). + */ + credits?: number; + subPlan?: string; + creditsPerRun?: number; + /** + * The caller's organization (org-based billing subject). Absent-safe: + * rendered as a single `org:` line when present (V3-routed callers only). + */ + activeOrg?: { + id: string; + name: string; + plan: string; + role: string; + remaining: number; + includedCredits: number; + seats: number; + }; + /** + * Every organization the underlying user belongs to (account-wide + * membership list, personal org included). Independent of `org` below. + * Absent-safe: omitted on a server-side lookup failure or an older + * backend. + */ + organizations?: CliOrgSummary[]; + /** + * The calling key's own org binding. Present only when the request + * authenticated with a Postgres-backed membership key (`sk-member-…`); + * absent for a legacy envelope key, which has no binding to echo. + */ + org?: CliOrgBinding; } export interface AuthDeps { @@ -175,6 +212,10 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): stderr(`[advisory] Inheriting api_url from existing profile: ${resolvedFromProfile}`); } + // Reject a malformed key up front — before any network — so `setup` fails + // fast with a VALIDATION_ERROR instead of a live ping or a raw Headers error. + assertValidApiKey(apiKey); + // Verify the key is accepted before persisting. Build an HttpClient // directly (bypassing loadConfig) so we can test the candidate key+url // before it is written to disk. This ensures we never overwrite a @@ -276,8 +317,26 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi `endpoint: ${resolvedEndpoint}`, `env: ${m.env}`, `scopes: ${m.scopes.join(', ')}`, + // Org context — confirms whose wallet a billable command will draw + // from, so it renders only for V3-routed callers (`v3Enabled` is the + // authoritative routing bit; field presence alone is not trusted). + ...(m.v3Enabled === true && m.activeOrg + ? [`org: ${m.activeOrg.name} (${m.activeOrg.plan}, ${m.activeOrg.role})`] + : []), // Authoritative routing mode, rendered only when the backend supplies it. ...(m.v3Enabled !== undefined ? [`routing: ${routingLabel(m.v3Enabled)}`] : []), + // Org attribution — account-wide membership list, rendered only when + // the backend supplies a non-empty list. + ...(formatOrgsSummary(m.organizations) + ? [`orgs: ${formatOrgsSummary(m.organizations)}`] + : []), + // The calling key's own org binding — membership keys only. + ...(formatOrgBinding(m.org) ? [`org binding: ${formatOrgBinding(m.org)}`] : []), + // A personal-scoped key held by someone who is also in a team: say so + // here rather than letting the team's projects just not show up. + ...(formatPersonalScopeHint(m.organizations, m.org) + ? [`note: ${formatPersonalScopeHint(m.organizations, m.org)}`] + : []), ]; // C2: warn in text mode when key cannot write/run const missingScopes = (['write:tests', 'run:tests'] as const).filter( diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index 7974947..80f8c73 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -73,7 +73,7 @@ beforeEach(() => { describe('runDoctor — healthy environment', () => { it('returns an all-passing report and does not throw', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const report = await runDoctor( { profile: 'default', output: 'text', debug: false }, @@ -88,7 +88,7 @@ describe('runDoctor — healthy environment', () => { }); it('adds a Routing check (v3) and the gap advisory when /me reports v3Enabled', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const report = await runDoctor( { profile: 'default', output: 'text', debug: false }, @@ -102,11 +102,11 @@ describe('runDoctor — healthy environment', () => { expect(report.failures).toBe(0); expect(report.checks.some(c => c.name === 'Routing' && c.detail.includes('v3'))).toBe(true); expect(capture.stderr.join('\n')).toContain('[advisory]'); - expect(capture.stderr.join('\n')).toContain('test cancel'); + expect(capture.stderr.join('\n')).toContain('--target-url'); }); it('shows Routing v2 and no advisory when v3Enabled is false', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const report = await runDoctor( { profile: 'default', output: 'text', debug: false }, @@ -122,7 +122,7 @@ describe('runDoctor — healthy environment', () => { }); it('omits the Routing check when /me does not report v3Enabled', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { deps } = makeCapture(); const report = await runDoctor( { profile: 'default', output: 'text', debug: false }, @@ -132,19 +132,58 @@ describe('runDoctor — healthy environment', () => { expect(report.warnings).toBe(0); }); + it('adds Organizations and Org binding checks when /me reports them', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { + ...healthyDeps(credentialsPath, { + fetchImpl: makeFetch({ + ...OK_ME, + organizations: [{ id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }], + org: { id: 'org_1', name: 'Acme Corp', role: 'owner' }, + }), + }), + ...deps, + }, + ); + expect(report.failures).toBe(0); + const orgsCheck = report.checks.find(c => c.name === 'Organizations'); + expect(orgsCheck?.status).toBe('ok'); + expect(orgsCheck?.detail).toBe('Acme Corp (org_1, role: owner)'); + const bindingCheck = report.checks.find(c => c.name === 'Org binding'); + expect(bindingCheck?.status).toBe('ok'); + expect(bindingCheck?.detail).toBe('Acme Corp (org_1, role: owner)'); + expect(capture.stdout.join('\n')).toContain('Organizations'); + expect(capture.stdout.join('\n')).toContain('Org binding'); + }); + + it('omits Organizations and Org binding checks when /me does not report them (older backend)', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, // OK_ME has no organizations/org + ); + expect(report.checks.some(c => c.name === 'Organizations')).toBe(false); + expect(report.checks.some(c => c.name === 'Org binding')).toBe(false); + expect(report.warnings).toBe(0); + }); + it('never prints the API key anywhere in the report', async () => { - writeProfile('default', { apiKey: 'sk-super-secret-value' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-super-secret-value' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runDoctor( { profile: 'default', output: 'text', debug: false }, { ...healthyDeps(credentialsPath), ...deps }, ); const all = capture.stdout.join('\n') + capture.stderr.join('\n'); - expect(all).not.toContain('sk-super-secret-value'); + expect(all).not.toContain('sk-user-super-secret-value'); }); it('emits a machine-readable report under --output json without leaking the API key', async () => { - writeProfile('default', { apiKey: 'sk-json-secret-value' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-json-secret-value' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runDoctor( { profile: 'default', output: 'json', debug: false }, @@ -153,7 +192,7 @@ describe('runDoctor — healthy environment', () => { const raw = capture.stdout.join(''); // Security: the JSON serialization path is distinct from the text renderer, // so assert the key never leaks here either. - expect(raw).not.toContain('sk-json-secret-value'); + expect(raw).not.toContain('sk-user-json-secret-value'); const parsed = JSON.parse(raw) as DoctorReport; expect(parsed.failures).toBe(0); expect(Array.isArray(parsed.checks)).toBe(true); @@ -178,7 +217,7 @@ describe('runDoctor — failing checks exit non-zero', () => { }); it('invalid endpoint URL fails the API endpoint check', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const rejection = await runDoctor( { profile: 'default', output: 'text', debug: false, endpointUrl: 'not-a-url' }, @@ -191,7 +230,7 @@ describe('runDoctor — failing checks exit non-zero', () => { }); it('rejected API key surfaces as a Connectivity failure', async () => { - writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-bad' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const authError = { error: { code: 'AUTH_INVALID', message: 'Bad key.', requestId: 'req_x', details: {} }, @@ -207,7 +246,7 @@ describe('runDoctor — failing checks exit non-zero', () => { }); it('a non-auth /me error is reported as a Connectivity failure with its code', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const notFound = { error: { code: 'NOT_FOUND', message: 'nope', requestId: 'req_y', details: {} }, @@ -221,7 +260,7 @@ describe('runDoctor — failing checks exit non-zero', () => { }); it('an outdated Node runtime fails the Node.js check', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const rejection = await runDoctor( { profile: 'default', output: 'text', debug: false }, @@ -236,7 +275,7 @@ describe('runDoctor — failing checks exit non-zero', () => { describe('runDoctor — warnings do not fail', () => { it('missing verify skill is a warning, not a failure', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const report = await runDoctor( { profile: 'default', output: 'text', debug: false }, @@ -294,7 +333,7 @@ describe('createDoctorCommand wiring', () => { }); it('accepts valid --output modes through command wiring', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); for (const mode of ['text', 'json'] as const) { const { capture, deps } = makeCapture(); await makeDoctorProgram({ ...healthyDeps(credentialsPath), ...deps }).parseAsync([ diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 40c1ac8..d53520a 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -24,6 +24,8 @@ import { import { loadConfig } from '../lib/config.js'; import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; +import type { CliOrgBinding, CliOrgSummary } from '../lib/org-render.js'; +import { formatOrgBinding, formatOrgsSummary, formatPersonalScopeHint } from '../lib/org-render.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; @@ -51,6 +53,10 @@ interface MeIdentity { userId?: string; keyId?: string; v3Enabled?: boolean; + /** Account-wide membership list. Absent-safe (older backends omit it). */ + organizations?: CliOrgSummary[]; + /** The calling key's own org binding — membership keys only. */ + org?: CliOrgBinding; } export interface DoctorDeps { @@ -112,6 +118,22 @@ export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Pro }); } + // Org attribution lines — only when the backend reported them (no new call). + const orgsSummary = formatOrgsSummary(connectivity.organizations); + if (orgsSummary) { + checks.push({ name: 'Organizations', status: 'ok', detail: orgsSummary }); + } + const orgBinding = formatOrgBinding(connectivity.org); + if (orgBinding) { + checks.push({ name: 'Org binding', status: 'ok', detail: orgBinding }); + } + // Warn, not fail: the key works — it just cannot see the team's work, which + // otherwise looks like missing data rather than a scoping choice. + const personalScopeHint = formatPersonalScopeHint(connectivity.organizations, connectivity.org); + if (personalScopeHint) { + checks.push({ name: 'Workspace scope', status: 'warn', detail: personalScopeHint }); + } + checks.push(checkSkill(cwd, deps)); const failures = checks.filter(check => check.status === 'fail').length; @@ -198,7 +220,12 @@ async function checkConnectivity( opts: CommonOptions, deps: DoctorDeps, ctx: { hasKey: boolean; endpointOk: boolean }, -): Promise<{ check: DoctorCheck; v3Enabled?: boolean }> { +): Promise<{ + check: DoctorCheck; + v3Enabled?: boolean; + organizations?: CliOrgSummary[]; + org?: CliOrgBinding; +}> { const name = 'Connectivity'; if (opts.dryRun) return { check: { name, status: 'warn', detail: 'skipped under --dry-run' } }; if (!ctx.hasKey) @@ -218,6 +245,8 @@ async function checkConnectivity( return { check: { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }, v3Enabled: me.v3Enabled, + organizations: me.organizations, + org: me.org, }; } catch (error) { if (error instanceof ApiError) { diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 92e2095..55bfd28 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -179,7 +179,7 @@ describe('runInit — happy path (interactive)', () => { const { captured, deps } = makeCapture(); const { fs: agentFs } = makeMemFs(); const fetchMock = makeOkFetch(); - const secretPrompt = vi.fn(async () => 'sk-test-key'); + const secretPrompt = vi.fn(async () => 'sk-user-test-key'); await runInit(makeBaseOpts(), { ...deps, @@ -215,7 +215,7 @@ describe('runInit — happy path (interactive)', () => { const { fs: agentFs } = makeMemFs(); const fetchMock = makeOkFetch(); - await runInit(makeBaseOpts({ output: 'json', apiKey: 'sk-json-test' }), { + await runInit(makeBaseOpts({ output: 'json', apiKey: 'sk-user-json-test' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -264,7 +264,7 @@ describe('runInit — happy path (interactive)', () => { }) as unknown as InitDeps['fetchImpl']; await runInit( - makeBaseOpts({ apiKey: 'sk-json-test', debug: true, noAgent: true, output: 'json' }), + makeBaseOpts({ apiKey: 'sk-user-json-test', debug: true, noAgent: true, output: 'json' }), { ...deps, fetchImpl: fetchMock, @@ -290,7 +290,7 @@ describe('runInit — --yes --api-key (non-interactive)', () => { const fetchMock = makeOkFetch(); const secretPrompt = vi.fn(async () => 'should-never-be-called'); - await runInit(makeBaseOpts({ apiKey: 'sk-test', yes: true }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-test', yes: true }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -319,7 +319,7 @@ describe('runInit — --no-agent', () => { const { fs: agentFs, writeCalls } = makeMemFs(); const fetchMock = makeOkFetch(); - await runInit(makeBaseOpts({ apiKey: 'sk-test', noAgent: true, output: 'json' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-test', noAgent: true, output: 'json' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -341,7 +341,7 @@ describe('runInit — --no-agent', () => { const { fs: agentFs } = makeMemFs(); const fetchMock = makeOkFetch(); - await runInit(makeBaseOpts({ apiKey: 'sk-test', noAgent: true }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-test', noAgent: true }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -368,7 +368,7 @@ describe('runInit — --no-agent', () => { const { fs: agentFs } = makeMemFs(); const fetchMock = makeOkFetch(); - await runInit(makeBaseOpts({ apiKey: 'sk-test' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-test' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -395,7 +395,7 @@ describe('runInit — default claude target installs 2 skill files', () => { const { fs: agentFs, writeCalls } = makeMemFs(); const fetchMock = makeOkFetch(); - await runInit(makeBaseOpts({ apiKey: 'sk-test' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-test' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -424,7 +424,7 @@ describe('runInit — --agent cursor', () => { const { fs: agentFs, writeCalls } = makeMemFs(); const fetchMock = makeOkFetch(); - await runInit(makeBaseOpts({ apiKey: 'sk-test', agent: 'cursor' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-test', agent: 'cursor' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -459,7 +459,7 @@ describe('runInit — --dry-run', () => { async () => new Response('{}', { status: 200 }), ) as unknown as InitDeps['fetchImpl']; - await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-dry' }), { + await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-user-dry' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -479,7 +479,7 @@ describe('runInit — --dry-run', () => { const { captured, deps } = makeCapture(); const { fs: agentFs } = makeMemFs(); - await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-dry' }), { + await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-user-dry' }), { ...deps, fetchImpl: vi.fn(async () => new Response('{}')) as unknown as InitDeps['fetchImpl'], credentialsPath, @@ -497,7 +497,7 @@ describe('runInit — --dry-run', () => { const { captured, deps } = makeCapture(); const { fs: agentFs } = makeMemFs(); - await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-dry', output: 'json' }), { + await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-user-dry', output: 'json' }), { ...deps, fetchImpl: vi.fn(async () => new Response('{}')) as unknown as InitDeps['fetchImpl'], credentialsPath, @@ -520,14 +520,17 @@ describe('runInit — --dry-run', () => { const { fs: agentFs, writeCalls } = makeMemFs(); const fetchMock = vi.fn(async () => new Response('{}')) as unknown as InitDeps['fetchImpl']; - await runInit(makeBaseOpts({ dryRun: true, apiKey: 'sk-dry', noAgent: true, output: 'json' }), { - ...deps, - fetchImpl: fetchMock, - credentialsPath, - isTTY: false, - cwd: CWD, - fs: agentFs, - }); + await runInit( + makeBaseOpts({ dryRun: true, apiKey: 'sk-user-dry', noAgent: true, output: 'json' }), + { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + cwd: CWD, + fs: agentFs, + }, + ); expect(fetchMock).not.toHaveBeenCalled(); expect(writeCalls).toHaveLength(0); @@ -587,7 +590,7 @@ describe('runInit — codex-review hardening', () => { const fetchImpl = makeOkFetch(); // env has NO TESTSPRITE_API_KEY; if --from-env wrongly won, runConfigure would // read undefined and throw. Success proves --api-key took precedence. - await runInit(makeBaseOpts({ apiKey: 'sk-wins', fromEnv: true, noAgent: true }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-wins', fromEnv: true, noAgent: true }), { ...deps, env: {}, fetchImpl, @@ -611,7 +614,7 @@ describe('runInit — codex-review hardening', () => { }), { ...deps, - env: { TESTSPRITE_API_KEY: 'sk' }, + env: { TESTSPRITE_API_KEY: 'sk-user-min' }, fetchImpl, credentialsPath, isTTY: false, @@ -634,7 +637,7 @@ describe('runInit — codex-review hardening', () => { // production/no-email banner even though configure wrote the correct key. const fetchImpl = vi.fn(async (_url: string, init: { headers?: Record }) => { const key = init.headers?.['x-api-key'] ?? init.headers?.['X-API-Key']; - if (key === 'sk-real') { + if (key === 'sk-user-real') { return new Response(JSON.stringify(ME), { status: 200, headers: { 'content-type': 'application/json' }, @@ -648,9 +651,9 @@ describe('runInit — codex-review hardening', () => { ); }) as unknown as InitDeps['fetchImpl']; - await runInit(makeBaseOpts({ apiKey: 'sk-real', noAgent: true, output: 'json' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-real', noAgent: true, output: 'json' }), { ...deps, - env: { TESTSPRITE_API_KEY: 'sk-stale-bogus' }, + env: { TESTSPRITE_API_KEY: 'sk-user-stale-bogus' }, fetchImpl, credentialsPath, isTTY: false, @@ -668,7 +671,7 @@ describe('runInit — codex-review hardening', () => { it('summary reports the endpoint from TESTSPRITE_API_URL, not a flat prod default', async () => { const { captured, deps } = makeCapture(); - await runInit(makeBaseOpts({ apiKey: 'sk-env-url', noAgent: true, output: 'json' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-env-url', noAgent: true, output: 'json' }), { ...deps, env: { TESTSPRITE_API_URL: 'https://api.example.com:8443' }, fetchImpl: makeOkFetch(), @@ -709,7 +712,7 @@ describe('runInit — bad API key', () => { let thrown: unknown; try { - await runInit(makeBaseOpts({ apiKey: 'sk-bad' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-bad' }), { ...deps, fetchImpl: makeAuthFailFetch(), credentialsPath, @@ -743,7 +746,7 @@ describe('runInit — summary JSON shape', () => { const { fs: agentFs } = makeMemFs(); const fetchMock = makeOkFetch(); - await runInit(makeBaseOpts({ apiKey: 'sk-shape', output: 'json' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-shape', output: 'json' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -766,7 +769,7 @@ describe('runInit — summary JSON shape', () => { const { fs: agentFs } = makeMemFs(); const fetchMock = makeOkFetch(); - await runInit(makeBaseOpts({ apiKey: 'sk-agent-shape', output: 'json' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-agent-shape', output: 'json' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -805,7 +808,7 @@ describe('runInit — --from-env', () => { ...deps, fetchImpl: fetchMock, credentialsPath, - env: { TESTSPRITE_API_KEY: 'sk-from-env-key' }, + env: { TESTSPRITE_API_KEY: 'sk-user-from-env-key' }, prompt: { secret: secretPrompt }, isTTY: false, cwd: CWD, @@ -840,7 +843,7 @@ describe('runInit — all agent targets', () => { 'credentials', ); - await runInit(makeBaseOpts({ apiKey: 'sk-target', agent: target }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-target', agent: target }), { ...deps, fetchImpl: fetchMock, credentialsPath: localCreds, @@ -889,14 +892,17 @@ describe('[B-E2E-05] runInit: --no-agent + --agent conflict emits [warn] on stde // Pass rawArgConflict signal: noAgent=true wins (--no-agent was last) // runInit exposes a rawArgConflict option that the command action passes // when it detects both --agent and --no-agent in rawArgs. - await runInit(makeBaseOpts({ apiKey: 'sk-conflict', noAgent: true, rawArgConflict: true }), { - ...deps, - fetchImpl: fetchMock, - credentialsPath: localCreds, - isTTY: false, - cwd: CWD, - fs: agentFs, - }); + await runInit( + makeBaseOpts({ apiKey: 'sk-user-conflict', noAgent: true, rawArgConflict: true }), + { + ...deps, + fetchImpl: fetchMock, + credentialsPath: localCreds, + isTTY: false, + cwd: CWD, + fs: agentFs, + }, + ); const warnLine = captured.stderr.find(l => l.includes('[warn]') && l.includes('--no-agent')); expect(warnLine).toBeDefined(); @@ -911,7 +917,7 @@ describe('[B-E2E-05] runInit: --no-agent + --agent conflict emits [warn] on stde await runInit( makeBaseOpts({ - apiKey: 'sk-conflict2', + apiKey: 'sk-user-conflict2', agent: 'cursor', noAgent: false, rawArgConflict: true, @@ -942,7 +948,7 @@ describe('[B-E2E-05] runInit: --no-agent + --agent conflict emits [warn] on stde await runInit( // rawArgConflict not set (default undefined/false) - makeBaseOpts({ apiKey: 'sk-no-conflict', agent: 'claude' }), + makeBaseOpts({ apiKey: 'sk-user-no-conflict', agent: 'claude' }), { ...deps, fetchImpl: fetchMock, @@ -990,7 +996,7 @@ describe('[B-E2E-06] runInit: install failure → info message on stderr + re-th let caughtErr: unknown; try { - await runInit(makeBaseOpts({ apiKey: 'sk-install-fail', agent: 'claude' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-install-fail', agent: 'claude' }), { ...deps, fetchImpl: fetchMock, credentialsPath: localCreds, @@ -1031,7 +1037,7 @@ describe('runInit — telemetry attribution (X-CLI-Command)', () => { }); }) as unknown as InitDeps['fetchImpl']; - await runInit(makeBaseOpts({ apiKey: 'sk-tag', noAgent: true, output: 'json' }), { + await runInit(makeBaseOpts({ apiKey: 'sk-user-tag', noAgent: true, output: 'json' }), { ...deps, fetchImpl: fetchMock, credentialsPath, @@ -1084,7 +1090,7 @@ describe('runInit -- skipIfConfigured', () => { const { fs: agentFs } = makeMemFs(); // No pre-existing credentials -- skip has no effect. const fetchMock = makeOkFetch(); - const prompt = { secret: vi.fn(async () => 'sk-fresh') }; + const prompt = { secret: vi.fn(async () => 'sk-user-fresh') }; await runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'text' }), { ...deps, @@ -1097,7 +1103,7 @@ describe('runInit -- skipIfConfigured', () => { // With no saved key, the prompt should fire. expect(prompt.secret).toHaveBeenCalledTimes(1); - expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-fresh'); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-user-fresh'); expect(captured.stdout.join('')).toContain('initialized'); }); @@ -1128,7 +1134,12 @@ describe('runInit -- skipIfConfigured', () => { const fetchMock = makeOkFetch(); await runInit( - makeBaseOpts({ apiKey: 'sk-new', skipIfConfigured: true, noAgent: true, output: 'text' }), + makeBaseOpts({ + apiKey: 'sk-user-new', + skipIfConfigured: true, + noAgent: true, + output: 'text', + }), { ...deps, credentialsPath, @@ -1139,6 +1150,6 @@ describe('runInit -- skipIfConfigured', () => { ); // Explicit --api-key must overwrite regardless of skipIfConfigured. - expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-new'); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-user-new'); }); }); diff --git a/src/commands/init.ts b/src/commands/init.ts index 659945f..3db346c 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -489,8 +489,8 @@ function resolveCommonOptions(command: Command): CommonOptions { const SETUP_DESCRIPTION = 'Set up TestSprite: configure your API key and install the TestSprite agent skills for your coding agent'; -/** Raw Commander options shared by `setup` and the deprecated `init` alias. */ -interface SetupCmdOpts { +/** Raw Commander options shared by `setup` and the deprecated `init`/`auth configure` aliases. */ +export interface SetupCmdOpts { apiKey?: string; fromEnv?: boolean; /** @@ -506,8 +506,8 @@ interface SetupCmdOpts { skipIfConfigured?: boolean; } -/** Attach the onboarding flags shared by `setup` and the `init` alias. */ -function addSetupOptions( +/** Attach the onboarding flags shared by `setup` and the `init`/`auth configure` aliases. */ +export function addSetupOptions( cmd: Command, validTargets: AgentTarget[], defaultAgent: AgentTarget, @@ -632,11 +632,17 @@ export function createDeprecatedInitCommand(deps: InitDeps = {}): Command { * consolidation, `auth configure` now runs FULL setup (configure + install) * so an agent that reaches for the old command still ends up with the skill. * `setup` is the ONLY path that writes credentials. + * + * Accepts the SAME `SetupCmdOpts` shape `setup` does — the alias previously + * only wired up `--from-env`, so README's "runs the full setup" claim didn't + * hold: `--yes`/`--agent`/`--api-key`/`--force`/`--dir`/`--no-agent` were all + * rejected as unknown options. `index.ts` attaches the full flag set via + * `addSetupOptions` before wiring this action. */ export async function runConfigureViaSetup( command: Command, deps: InitDeps, - fromEnv: boolean, + cmdOpts: SetupCmdOpts, ): Promise { - await runSetupAction({ agent: 'claude', fromEnv }, command, deps, 'claude'); + await runSetupAction(cmdOpts, command, deps, 'claude'); } diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index ae98070..60a1671 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -6,6 +6,7 @@ import { ApiError } from '../lib/errors.js'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; import { type CliProject, + type CliCreateProjectResponse, type CliDeleteProjectResponse, type CliUpdateProjectResponse, createProjectCommand, @@ -399,6 +400,165 @@ describe('runList', () => { }); }); +describe('runList — org attribution (ORG column)', () => { + it('adds the ORG column only when at least one row carries orgId', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { + items: [{ ...PROJECT_FIXTURE, orgId: 'org_1', orgName: 'Acme Corp' }], + nextToken: null, + }, + })); + + const out: string[] = []; + await runList( + { profile: 'default', output: 'text', debug: false, pageSize: 25 }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const block = out.join('\n'); + expect(block).toContain('ORG'); + expect(block).toContain('Acme Corp'); + }); + + it('omits the ORG column entirely for a legacy (non-org-scoped) response', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { profile: 'default', output: 'text', debug: false, pageSize: 25 }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(out.join('\n')).not.toContain('ORG'); + }); + + it('falls back to orgId when orgName is absent', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [{ ...PROJECT_FIXTURE, orgId: 'org_1' }], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { profile: 'default', output: 'text', debug: false, pageSize: 25 }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const block = out.join('\n'); + expect(block).toContain('ORG'); + expect(block).toContain('org_1'); + }); + + it('an explicit --columns org still renders the column on a legacy (no-org-data) page', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + pageSize: 25, + columns: 'name,org', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + // No org data on this row -> the ORG cell renders empty, but the column + // (and the explicit selection) is still honored, not rejected. + expect(out.join('\n')).toMatch(/^Checkout\s*$/); + }); + + it('JSON output passes orgId/orgName through verbatim', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { + items: [{ ...PROJECT_FIXTURE, orgId: 'org_1', orgName: 'Acme Corp' }], + nextToken: null, + }, + })); + + const out: string[] = []; + await runList( + { profile: 'default', output: 'json', debug: false, pageSize: 25 }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const parsed = JSON.parse(out.join('\n')) as { items: CliProject[] }; + expect(parsed.items[0]!.orgId).toBe('org_1'); + expect(parsed.items[0]!.orgName).toBe('Acme Corp'); + }); +}); + +describe('runGet — org attribution', () => { + it('renders an `org:` line when the project carries orgId/orgName', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { ...PROJECT_FIXTURE, orgId: 'org_1', orgName: 'Acme Corp' }, + })); + + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, projectId: PROJECT_FIXTURE.id }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(out.join('\n')).toContain('org: Acme Corp (org_1)'); + }); + + it('falls back to "(name unknown)" when orgId is present but orgName is absent', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { ...PROJECT_FIXTURE, orgId: 'org_1' }, + })); + + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, projectId: PROJECT_FIXTURE.id }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(out.join('\n')).toContain('org: (name unknown) (org_1)'); + }); + + it('omits the `org:` line entirely when the project has no org attribution', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: PROJECT_FIXTURE })); + + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, projectId: PROJECT_FIXTURE.id }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(out.join('\n')).not.toContain('org:'); + expect(out.join('\n')).not.toContain('undefined'); + }); + + it('JSON output passes orgId/orgName through verbatim', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { ...PROJECT_FIXTURE, orgId: 'org_1', orgName: 'Acme Corp' }, + })); + + const project = await runGet( + { profile: 'default', output: 'json', debug: false, projectId: PROJECT_FIXTURE.id }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + + expect(project.orgId).toBe('org_1'); + expect(project.orgName).toBe('Acme Corp'); + }); +}); + describe('DEV-244 — project update no longer accepts the dead --description flag', () => { it('rejects --description on `project update` as an unknown option', async () => { const project = createProjectCommand(); @@ -484,6 +644,70 @@ describe('runGet', () => { expect(block).toContain('createdFrom: portal'); }); + it('renders a configured targetUrl', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { ...PROJECT_FIXTURE, targetUrl: 'https://staging.example.com' }, + })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, projectId: 'project_b3c91efa' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).toContain('targetUrl: https://staging.example.com'); + }); + + it('renders an explicit null targetUrl as "(not set)" with the fix-it command', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: { ...PROJECT_FIXTURE, targetUrl: null } })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, projectId: 'project_b3c91efa' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('targetUrl: (not set'); + expect(block).toContain('testsprite project update project_b3c91efa --url '); + }); + + it('offers the same fix-it for a BACKEND project with a null targetUrl (the V3 case, where it works)', async () => { + // A null on a backend project only ever comes from V3, where the default + // environment URL is real, settable, and exactly what the run guard checks — + // so the `--url` remedy is correct there. V2 never sends null for a backend + // project (it omits the key, because a V2 backend run resolves no project URL + // and the remedy would change nothing) — which is why this renderer can print + // one message for `null` without asking which execution path it came from. + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { ...PROJECT_FIXTURE, type: 'backend', targetUrl: null }, + })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, projectId: 'project_b3c91efa' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('type: backend'); + expect(block).toContain('testsprite project update project_b3c91efa --url '); + }); + + // NOTE: a guard, not a proof — this assertion also holds on the pre-change + // renderer (which had no targetUrl line at all). It exists so a future edit + // cannot switch the presence check to a truthiness check without going red. + it('says nothing about targetUrl when the backend omits the field (older backend, or `project list`)', async () => { + const { credentialsPath } = makeCreds(); + // PROJECT_FIXTURE deliberately has no targetUrl key — an absent field must + // NOT render as "(not set)", or every project on a pre-field backend reads + // as URL-less, including the ones that have a URL. + const fetchImpl = makeFetch(() => ({ body: PROJECT_FIXTURE })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, projectId: 'project_b3c91efa' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + expect(out.join('\n')).not.toContain('targetUrl'); + }); + it('NOT_FOUND envelope from server propagates as ApiError exit 4', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ @@ -779,6 +1003,250 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + + describe('backend project without --url — dead-on-arrival advisory (dogfood 2026-07-30)', () => { + it('emits [advisory] on stderr naming no-target-resolvable + the copy-pasteable fix, with the real created project id', async () => { + const { credentialsPath } = makeCreds(); + const createdProject: CliProject = { + ...PROJECT_FIXTURE, + id: 'proj_be_nourl', + type: 'backend', + name: 'No URL BE', + }; + const fetchImpl = makeFetch(() => ({ body: createdProject })); + const stderrLines: string[] = []; + + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + type: 'backend', + name: 'No URL BE', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + ); + + const advisory = stderrLines.find(l => l.includes('[advisory]')); + expect(advisory).toBeDefined(); + expect(advisory).toContain('no-target-resolvable'); + expect(advisory).toContain('testsprite project update proj_be_nourl --url '); + // The guard that produces no-target-resolvable only applies on the V3 + // execution path (V2 backend runs never resolve a project URL at all), + // so the claim must be scoped, not stated as universal — and it must + // point the reader at how to check which path they're on. + expect(advisory).toContain('V3'); + expect(advisory).toContain('auth status'); + }); + + it('does NOT emit the advisory when --url is supplied for a backend project', async () => { + const { credentialsPath } = makeCreds(); + const createdProject: CliProject = { + ...PROJECT_FIXTURE, + id: 'proj_be_withurl', + type: 'backend', + name: 'With URL BE', + }; + const fetchImpl = makeFetch(() => ({ body: createdProject })); + const stderrLines: string[] = []; + + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + type: 'backend', + name: 'With URL BE', + targetUrl: 'https://staging.example.com', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + ); + + expect( + stderrLines.some(l => l.includes('[advisory]') && l.includes('no-target-resolvable')), + ).toBe(false); + }); + + it('does NOT emit the advisory for a frontend project (--url is already required there)', async () => { + const { credentialsPath } = makeCreds(); + const createdProject: CliProject = { + ...PROJECT_FIXTURE, + id: 'proj_fe', + type: 'frontend', + name: 'FE Project', + }; + const fetchImpl = makeFetch(() => ({ body: createdProject })); + const stderrLines: string[] = []; + + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + type: 'frontend', + name: 'FE Project', + targetUrl: 'https://staging.example.com', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + ); + + expect( + stderrLines.some(l => l.includes('[advisory]') && l.includes('no-target-resolvable')), + ).toBe(false); + }); + + it('emits the advisory to stderr in --output json mode too, while stdout stays valid parseable JSON with no advisory text', async () => { + // The advisory goes to stderr, which no output mode ever routes into + // stdout — there is nothing for a --output json gate to protect, and + // --output json is exactly the non-interactive/agent/CI case where a + // silent dead-on-arrival project is most costly (nobody is watching a + // terminal for a warning that never fires). + const { credentialsPath } = makeCreds(); + const createdProject: CliProject = { + ...PROJECT_FIXTURE, + id: 'proj_be_json', + type: 'backend', + name: 'JSON BE', + }; + const fetchImpl = makeFetch(() => ({ body: createdProject })); + const stderrLines: string[] = []; + const stdoutLines: string[] = []; + + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'backend', + name: 'JSON BE', + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + }, + ); + + expect(result.type).toBe('backend'); + expect( + stderrLines.some(l => l.includes('[advisory]') && l.includes('no-target-resolvable')), + ).toBe(true); + + // Pipeline-contract assertion: stdout is still exactly one valid, + // parseable JSON payload — the advisory never leaks into it. + expect(stdoutLines).toHaveLength(1); + const parsed: unknown = JSON.parse(stdoutLines[0]!); + expect((parsed as CliCreateProjectResponse).type).toBe('backend'); + expect(stdoutLines[0]).not.toContain('[advisory]'); + }); + + it('does NOT emit the advisory under --dry-run (the remedy would embed the fake dry-run project id)', async () => { + resetDryRunBannerForTesting(); + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network in dry-run'); + }); + const stderrLines: string[] = []; + + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: true, + type: 'backend', + name: 'DryRun BE No URL', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect( + stderrLines.some(l => l.includes('[advisory]') && l.includes('no-target-resolvable')), + ).toBe(false); + }); + }); + + describe('id-field normalization', () => { + it('backfills `id` when the live response only carries `projectId`', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { + projectId: 'proj_live_shape', + type: 'frontend', + name: 'Live Shape Project', + createdFrom: 'cli', + createdAt: '2026-07-16T00:00:00.000Z', + }, + })); + + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Live Shape Project', + targetUrl: 'https://example.com', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(result.projectId).toBe('proj_live_shape'); + expect(result.id).toBe('proj_live_shape'); + }); + + it('backfills `projectId` when the live response only carries `id` (pre-fix shape)', async () => { + const { credentialsPath } = makeCreds(); + const createdProject: CliProject = { + ...PROJECT_FIXTURE, + id: 'proj_legacy_shape', + }; + const fetchImpl = makeFetch(() => ({ body: createdProject })); + + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Legacy Shape Project', + targetUrl: 'https://example.com', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(result.id).toBe('proj_legacy_shape'); + expect(result.projectId).toBe('proj_legacy_shape'); + }); + + it('--dry-run sample teaches both id field names', async () => { + resetDryRunBannerForTesting(); + const { credentialsPath } = makeCreds(); + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + type: 'frontend', + name: 'DryRun Shape Project', + targetUrl: 'https://example.com', + }, + { credentialsPath, stdout: () => {}, stderr: () => {} }, + ); + + expect(typeof result.projectId).toBe('string'); + expect(typeof result.id).toBe('string'); + expect(result.projectId).toBe(result.id); + }); + }); }); // --------------------------------------------------------------------------- @@ -1042,6 +1510,77 @@ describe('runUpdate', () => { expect(result.id).toBe('proj_json_no_fields'); expect(result.updatedFields).toBeUndefined(); }); + + describe('id-field normalization', () => { + it('backfills `id` when the live response only carries `projectId`', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { + projectId: 'proj_live_update_shape', + updatedFields: ['name'], + updatedAt: '2026-07-16T00:00:00.000Z', + }, + })); + + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_live_update_shape', + name: 'New Name', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(result.projectId).toBe('proj_live_update_shape'); + expect(result.id).toBe('proj_live_update_shape'); + }); + + it('backfills `projectId` when the live response only carries `id` (pre-fix shape)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { + id: 'proj_legacy_update_shape', + updatedFields: ['name'], + updatedAt: '2026-07-16T00:00:00.000Z', + }, + })); + + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_legacy_update_shape', + name: 'New Name', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(result.id).toBe('proj_legacy_update_shape'); + expect(result.projectId).toBe('proj_legacy_update_shape'); + }); + + it('--dry-run sample teaches both id field names', async () => { + resetDryRunBannerForTesting(); + const { credentialsPath } = makeCreds(); + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'proj_dryrun_update_shape', + name: 'New Name', + }, + { credentialsPath, stdout: () => {}, stderr: () => {} }, + ); + + expect(result.projectId).toBe('proj_dryrun_update_shape'); + expect(result.id).toBe('proj_dryrun_update_shape'); + }); + }); }); describe('runDelete', () => { diff --git a/src/commands/project.ts b/src/commands/project.ts index 6a0b332..fd0c15b 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -29,6 +29,38 @@ export interface CliProject { createdFrom: 'portal' | 'mcp' | 'cli'; createdAt: string; updatedAt: string; + /** + * Owning organization id + human-readable name. Additive + absent-safe: + * populated only for a membership-key (`sk-member-…`) caller on `project list` + * (org attribution across the caller's org-scoped view); `project get` + * does not populate them today even for a bound key. `orgName` may be + * absent even when `orgId` is present (best-effort name lookup). Legacy + * (unbound) callers never see either field. + */ + orgId?: string; + orgName?: string; + /** + * The project's default target/environment URL, or `null` when the project has + * none configured. + * + * **Absent vs. `null` is load-bearing here.** `null` means "the server resolved + * it and this project has no URL configured" — actionable, so the renderer + * prints the `project update … --url` remedy. **Absent** means "no answer", and + * the server uses it for every case where a remedy would be a lie: an older + * backend, the `list` endpoint (which would pay an extra read per row), a + * resolution that failed (a transient outage must never be reported as "no URL + * set"), and a V2 backend project (V2 backend runs resolve no project URL at + * all, so there is nothing to report and nothing `--url` would change). + * + * So the renderer must key on presence, not truthiness: printing "(not set)" + * for an absent field would report every project on an older backend — and + * every project in a `list` — as having no URL, including the ones that do. + * + * A backend project with no URL is not merely cosmetic: on the V3 execution + * path the first run of any of its tests is rejected `no-target-resolvable` + * (see the `project create --type backend` note in CLAUDE.md). + */ + targetUrl?: string | null; } export interface ProjectDeps { @@ -123,7 +155,37 @@ export interface CliCreateProjectRequest { instruction?: string; } -export type CliCreateProjectResponse = CliProject; +/** + * Response shape for `POST /projects`. + * + * The validation sweep found the LIVE `POST /projects` + * response keying its id as `projectId` (matching `CliDeleteProjectResponse`'s + * convention), not `id` like the read paths (`GET /projects`, `GET + * /projects/{id}` — those are proven-`id` via the dev-e2e smoke test) — and + * possibly omitting `targetUrl`/`updatedAt` entirely. Rather than guess which + * single shape is "the real one" and risk flipping the drift instead of + * fixing it, both id field names are accepted here (and normalized — see + * `resolveCreatedProjectId` — so JSON consumers keyed on either `id` or + * `projectId` keep working), and `targetUrl`/`updatedAt` are optional. + */ +export interface CliCreateProjectResponse { + /** Preferred — matches the live backend's response and `project delete`'s `projectId`. */ + projectId?: string; + /** Legacy/fallback name; some responses (and the read paths) use this instead. */ + id?: string; + name: string; + type: 'frontend' | 'backend'; + createdFrom: 'portal' | 'mcp' | 'cli'; + createdAt: string; + /** Absent-safe: not guaranteed on every backend response. */ + updatedAt?: string; + targetUrl?: string; +} + +/** Resolve the created project's id regardless of which field name the backend used. */ +export function resolveCreatedProjectId(r: CliCreateProjectResponse): string | undefined { + return r.projectId ?? r.id; +} interface CreateOptions extends CommonOptions { type: 'frontend' | 'backend'; @@ -194,7 +256,11 @@ export async function runCreate( ) { stderr(`idempotency-key: ${idempotencyKey}`); } + // Teach both id field names — `projectId` is the field the + // live backend actually sends; `id` is kept so this sample still matches + // callers written against the pre-fix shape. const sample: CliCreateProjectResponse = { + projectId: 'p_dryrun_2026', id: 'p_dryrun_2026', type: opts.type, name: opts.name, @@ -202,8 +268,8 @@ export async function runCreate( createdFrom: 'cli', createdAt: '2026-05-16T00:00:00.000Z', updatedAt: '2026-05-16T00:00:00.000Z', - } as unknown as CliCreateProjectResponse; - out.print(sample, data => renderProjectText(data as CliProject)); + }; + out.print(sample, data => renderCreateProjectText(data as CliCreateProjectResponse)); return sample; } @@ -228,12 +294,63 @@ export async function runCreate( }; const client = makeClient(opts, deps); - const created = await client.post('/projects', { + const rawCreated = await client.post('/projects', { body, headers: { 'idempotency-key': idempotencyKey }, }); + // Normalize whichever id field name the backend actually + // sent onto BOTH `projectId` and `id`, so JSON consumers keyed on either + // name keep working regardless of which one the live response used. + const resolvedId = resolveCreatedProjectId(rawCreated); + const created: CliCreateProjectResponse = { + ...rawCreated, + ...(resolvedId !== undefined ? { projectId: resolvedId, id: resolvedId } : {}), + }; + + out.print(created, data => renderCreateProjectText(data as CliCreateProjectResponse)); + + // A backend project created without --url has no default environment URL. + // On the V3 execution path, the shared admission guard (`runProjectGuarded` + // -> `assertProjectEnvNotLocal`, called by every V3 run/rerun/batch entry + // point, including `backendRun`) rejects the FIRST run of any test in the + // project with 400 no-target-resolvable. On the V2 path (`cli-run.service.ts` + // -> `resolveRunPrereqsFE`), this URL resolution is frontend-only and is + // never applied to a backend run, so a V2-routed backend project without a + // URL runs fine — V2 is the only execution path live in production today + // (V3 is dev-only pending its own GA release), so the wording below is + // conditioned on V3 rather than stated as a universal consequence. + // `--url` is intentionally NOT made mandatory here (that would break the + // published --help contract: npm 0.4.0 already ships text saying the URL + // is frontend-only), so this stays advisory-only. The CLI does not call + // GET /me here to check the caller's actual v3Enabled routing before + // deciding whether to print this — an extra round-trip on every create is + // not worth it just to gate a hint — so it points the reader at `auth + // status` (which already renders the routing: v2|v3 line) instead. + // Emitted in EVERY output mode, including --output json: this goes to + // stderr, which never touches JSON stdout, so there is nothing to protect + // by suppressing it (unlike `emitV3RoutingAdvisory`, which withholds + // account-context advisory in JSON mode because a JSON caller can read + // `v3Enabled` directly instead — there is no equivalent structured signal + // for this one). And --output json is precisely the case that needs it + // most: a script or agent creating a project non-interactively has no one + // watching a terminal, so a silent dead-on-arrival project only surfaces + // later as an unexplained 400 on the first run. Same family as the C1 + // `--target-url` advisory above (a flag/setup the caller just made has a + // structural consequence) rather than the routing-advisory family. + // Real path only — NOT duplicated into the dry-run branch above, because + // the remedy command below embeds the created project id, and under + // --dry-run that id is the canned `p_dryrun_2026` sample: emitting a + // live-looking fix-it command against a fake resource is exactly what the + // `dashboardUrl` convention (see CLAUDE.md) already decided to suppress. + if (opts.type === 'backend' && !opts.targetUrl) { + stderr( + `[advisory] this backend project has no default environment URL. On the V3 execution ` + + `path (check with \`testsprite auth status\`), test runs are rejected with ` + + `no-target-resolvable until one is set. Fix: testsprite project update ` + + `${resolvedId ?? ''} --url `, + ); + } - out.print(created, data => renderProjectText(data as CliProject)); return created; } @@ -241,11 +358,27 @@ export async function runCreate( // project update // --------------------------------------------------------------------------- +/** + * Response shape for `PATCH /projects/{id}`. + * + * Same `id`-field drift as `CliCreateProjectResponse` — both + * names are accepted and normalized (see `resolveUpdatedProjectId`), and + * `updatedAt` is optional since the live response may omit it. + */ export interface CliUpdateProjectResponse { - id: string; + /** Preferred — matches the live backend's response and `project delete`'s `projectId`. */ + projectId?: string; + /** Legacy/fallback name; some responses use this instead. */ + id?: string; /** Backend may omit this field; treat absence as no specific fields reported. */ updatedFields?: string[]; - updatedAt: string; + /** Absent-safe: not guaranteed on every backend response. */ + updatedAt?: string; +} + +/** Resolve the updated project's id regardless of which field name the backend used. */ +export function resolveUpdatedProjectId(r: CliUpdateProjectResponse): string | undefined { + return r.projectId ?? r.id; } interface UpdateOptions extends CommonOptions { @@ -313,7 +446,11 @@ export async function runUpdate( ) { stderr(`idempotency-key: ${idempotencyKey}`); } + // Teach both id field names — `projectId` is the field the + // live backend actually sends; `id` is kept so this sample still matches + // callers written against the pre-fix shape. const sample: CliUpdateProjectResponse = { + projectId: opts.projectId, id: opts.projectId, updatedFields: presentFieldNames, updatedAt: '2026-05-16T00:00:00.000Z', @@ -345,13 +482,23 @@ export async function runUpdate( Object.entries(bodyFields).filter(([, v]) => v !== undefined), ) as Record; const client = makeClient(opts, deps); - const updated = await client.patch( + const rawUpdated = await client.patch( `/projects/${encodeURIComponent(opts.projectId)}`, { body, headers: { 'idempotency-key': idempotencyKey }, }, ); + // Normalize whichever id field name the backend actually + // sent onto BOTH `projectId` and `id`, so JSON consumers keyed on either + // name keep working regardless of which one the live response used. + const resolvedUpdatedId = resolveUpdatedProjectId(rawUpdated); + const updated: CliUpdateProjectResponse = { + ...rawUpdated, + ...(resolvedUpdatedId !== undefined + ? { projectId: resolvedUpdatedId, id: resolvedUpdatedId } + : {}), + }; out.print(updated, data => renderUpdateText(data as CliUpdateProjectResponse)); return updated; @@ -704,7 +851,11 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .description('Create a new project') .option('--type ', 'project type (required)') .option('--name ', 'project name (required)') - .option('--url ', 'target URL (required for frontend)') + .option( + '--url ', + 'target URL (required for frontend; also required for backend on the V3 execution ' + + 'path — see `auth status` for your routing)', + ) .option( '--description ', 'not supported — projects have no description (test-level descriptions are set on `test create`)', @@ -1013,6 +1164,14 @@ function makeOutput(mode: OutputMode, deps: ProjectDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } +/** + * Master column set — includes ORG so `--columns` validation and explicit + * `--columns ...,org` selection both work regardless of whether any row in + * THIS response happens to carry org attribution. The default (no + * `--columns` flag) rendering uses {@link defaultProjectListColumns} instead, + * which drops ORG unless at least one row carries it — this keeps the table + * unchanged for legacy (non-org-scoped) callers. + */ const PROJECT_LIST_COLUMNS: ReadonlyArray> = [ { header: 'ID', @@ -1026,9 +1185,31 @@ const PROJECT_LIST_COLUMNS: ReadonlyArray> = [ }, { header: 'TYPE', width: 8, render: project => project.type }, { header: 'FROM', width: 6, render: project => project.createdFrom }, + { + header: 'ORG', + width: rows => + Math.max(3, ...rows.map(project => (project.orgName ?? project.orgId ?? '').length)), + render: project => project.orgName ?? project.orgId ?? '', + }, { header: 'CREATED', width: 0, render: project => project.createdAt }, ]; +const PROJECT_LIST_ORG_COLUMN = PROJECT_LIST_COLUMNS.find(c => c.header === 'ORG')!; + +/** + * Default (no explicit `--columns`) column set. ORG is included only when + * at least one row in this page carries `orgId` — avoids widening the table + * for callers whose projects have no org attribution at all. + */ +function defaultProjectListColumns( + rows: readonly CliProject[], +): ReadonlyArray> { + const hasOrgInfo = rows.some(project => project.orgId !== undefined); + return hasOrgInfo + ? PROJECT_LIST_COLUMNS + : PROJECT_LIST_COLUMNS.filter(c => c !== PROJECT_LIST_ORG_COLUMN); +} + function renderProjectListText( page: Page, options: { columns?: string; noHeader?: boolean } = {}, @@ -1038,8 +1219,13 @@ function renderProjectListText( ? `No projects on this page.\nnextToken: ${page.nextToken}` : 'No projects.'; } + // Explicit --columns: resolve against the FULL master set (so `org` can be + // requested even when this page happens to have no org-scoped rows). + // Default: only include ORG when the data actually carries it. + const columns = + options.columns !== undefined ? PROJECT_LIST_COLUMNS : defaultProjectListColumns(page.items); const lines = [ - renderTextTable(page.items, PROJECT_LIST_COLUMNS, { + renderTextTable(page.items, columns, { columns: options.columns, noHeader: options.noHeader, }), @@ -1049,22 +1235,55 @@ function renderProjectListText( } function renderProjectText(p: CliProject): string { - return [ + const lines = [ `id: ${p.id}`, `name: ${p.name}`, `type: ${p.type}`, `createdFrom: ${p.createdFrom}`, `createdAt: ${p.createdAt}`, `updatedAt: ${p.updatedAt}`, - ].join('\n'); + ]; + if (p.orgId !== undefined) { + lines.push(`org: ${p.orgName ?? '(name unknown)'} (${p.orgId})`); + } + // Presence, not truthiness — see the `targetUrl` docstring on `CliProject`. + // `'targetUrl' in p` distinguishes "the backend answered 'no URL'" (render the + // hint) from "this endpoint doesn't report it" (say nothing). + if ('targetUrl' in p) { + lines.push( + p.targetUrl + ? `targetUrl: ${p.targetUrl}` + : `targetUrl: (not set — set one with: testsprite project update ${p.id} --url )`, + ); + } + return lines.join('\n'); +} + +/** + * `project create`'s text renderer. Distinct from + * `renderProjectText` (used by `project get`/`list`, where the live `id` + * field is proven reliable) because the create response's id field name and + * `updatedAt` presence are not guaranteed — see `CliCreateProjectResponse`. + */ +function renderCreateProjectText(p: CliCreateProjectResponse): string { + const lines = [ + `id: ${resolveCreatedProjectId(p) ?? '(unknown)'}`, + `name: ${p.name}`, + `type: ${p.type}`, + `createdFrom: ${p.createdFrom}`, + `createdAt: ${p.createdAt}`, + ]; + if (p.updatedAt !== undefined) lines.push(`updatedAt: ${p.updatedAt}`); + return lines.join('\n'); } function renderUpdateText(r: CliUpdateProjectResponse): string { - return [ - `id: ${r.id}`, + const lines = [ + `id: ${resolveUpdatedProjectId(r) ?? '(unknown)'}`, `updatedFields: ${r.updatedFields?.join(', ') ?? '(none)'}`, - `updatedAt: ${r.updatedAt}`, - ].join('\n'); + ]; + if (r.updatedAt !== undefined) lines.push(`updatedAt: ${r.updatedAt}`); + return lines.join('\n'); } function renderDeleteText(r: CliDeleteProjectResponse): string { diff --git a/src/commands/test.flaky.spec.ts b/src/commands/test.flaky.spec.ts index 8076f3e..02c8b40 100644 --- a/src/commands/test.flaky.spec.ts +++ b/src/commands/test.flaky.spec.ts @@ -14,6 +14,7 @@ import { describe, expect, it } from 'vitest'; import { CLIError, ApiError } from '../lib/errors.js'; import type { FlakyReport } from '../lib/flaky.js'; import type { FetchImpl } from '../lib/http.js'; +import type { RerunAdvisory } from '../lib/runs.types.js'; import { runFlaky } from './test.js'; type FetchInput = Parameters[0]; @@ -48,8 +49,16 @@ function makeFlakyFetch(opts: { testType?: 'frontend' | 'backend'; notFoundOnTrigger?: boolean; triggerAuthError?: TriggerAuthErrorCode; -}): { fetchImpl: FetchImpl; triggerCount: () => number } { + /** Advisories echoed on every `POST /runs/rerun` response. */ + advisories?: RerunAdvisory[]; +}): { + fetchImpl: FetchImpl; + triggerCount: () => number; + /** Every rerun request body sent, in attempt order. */ + sentBodies: () => unknown[]; +} { let triggers = 0; + const sentBodies: unknown[] = []; const testType = opts.testType ?? 'frontend'; const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { const url = urlOf(input); @@ -69,6 +78,7 @@ function makeFlakyFetch(opts: { } if (method === 'POST' && url.includes('/runs/rerun')) { + sentBodies.push(init.body ? JSON.parse(init.body as string) : null); if (opts.triggerAuthError) { return jsonResponse(opts.triggerAuthError === 'AUTH_FORBIDDEN' ? 403 : 401, { error: { @@ -98,6 +108,7 @@ function makeFlakyFetch(opts: { enqueuedAt: '2026-06-03T10:00:00.000Z', codeVersion: 'v1', autoHeal: false, + ...(opts.advisories ? { advisories: opts.advisories } : {}), }); } @@ -132,7 +143,7 @@ function makeFlakyFetch(opts: { }); }) as FetchImpl; - return { fetchImpl, triggerCount: () => triggers }; + return { fetchImpl, triggerCount: () => triggers, sentBodies: () => sentBodies }; } function makeCreds(): { credentialsPath: string } { @@ -429,3 +440,192 @@ describe('runFlaky', () => { expect((err as ApiError).exitCode).toBe(5); }); }); + +// --------------------------------------------------------------------------- +// RATE_LIMITED during an attempt's poll — same defect class as the +// InterruptError branch: the HTTP layer already retried internally and gave +// up, but the attempt's run keeps executing (and billing) server-side. The +// current attempt's runId must be named on stderr before the whole probe +// aborts, and the exit code must stay 11 (never reclassified to 7/1). +// --------------------------------------------------------------------------- + +describe('runFlaky — RATE_LIMITED during an attempt poll', () => { + it('names the runId on stderr (stdout deliberately stays empty, matching the InterruptError branch) and rethrows with exit 11 (not 7)', async () => { + const fetchImpl: FetchImpl = (async (input: unknown, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const method = (init.method ?? 'GET').toUpperCase(); + + if (method === 'GET' && /\/tests\/[^/]+$/.test(url.split('?')[0]!)) { + return new Response( + JSON.stringify({ + id: 'test_x', + projectId: 'project_abc', + name: 'sample', + type: 'frontend', + createdFrom: 'portal', + status: 'passed', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + } + if (method === 'POST' && url.includes('/runs/rerun')) { + return new Response( + JSON.stringify({ + runId: 'run_1', + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + } + if (method === 'GET' && url.includes('/runs/run_1')) { + // retry-after: 0 keeps http.ts's internal retry-then-throw budget fast. + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: too many requests from this IP.', + nextAction: '', + requestId: 'req_rl_flaky_test', + details: {}, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '0' } }, + ); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }) as FetchImpl; + + const { deps, stdout, stderr } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('RATE_LIMITED'); + expect((err as ApiError).exitCode).toBe(11); + + const stderrBlock = stderr.join('\n'); + expect(stderrBlock).toContain('run_1'); + expect(stderrBlock).toContain('test wait'); + expect(stderrBlock).toContain('Rate limited'); + + // Deliberate, not an oversight: `test flaky` has no single-run partial + // envelope to emit (its output is an aggregate FlakyReport over N attempts, + // not a RunResponse) — this mirrors the neighboring InterruptError branch + // in the same catch block, which is also stderr-only. Pin it so a future + // reader doesn't "helpfully" add a stdout partial to only this branch and + // make the two adjacent detach reasons disagree on the output contract. + expect(stdout).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Flaky always sends an explicit autoHeal:false + surfaces advisories once +// --------------------------------------------------------------------------- + +describe('runFlaky — explicit autoHeal:false + advisories', () => { + it('sends autoHeal:false explicitly on every replay request', async () => { + const { fetchImpl, sentBodies } = makeFlakyFetch({ + statuses: ['passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ); + + const bodies = sentBodies(); + expect(bodies).toHaveLength(3); + for (const body of bodies) { + expect((body as { autoHeal?: boolean }).autoHeal).toBe(false); + expect((body as { source?: string }).source).toBe('cli'); + } + }); + + it('surfaces a repeated server advisory ONCE in the report/summary, not once per attempt', async () => { + const advisory = { + feature: 'autoHeal', + message: + 'The auto-heal opt-out was forwarded to the execution engine but is not yet enforced there.', + }; + const { fetchImpl } = makeFlakyFetch({ + statuses: ['passed', 'passed', 'passed'], + advisories: [advisory], + }); + const { deps, stderr } = makeDeps(fetchImpl); + const report = (await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + )) as FlakyReport; + + // Exactly one stderr line — even though 3 attempts each echoed the advisory. + const advisoryLines = stderr.filter(l => l === `[advisory] ${advisory.message}`); + expect(advisoryLines).toHaveLength(1); + + // Deduped into the report too, for JSON consumers. + expect(report.advisories).toEqual([advisory]); + }); + + it('absent advisories: report.advisories is empty and no advisory line is printed', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'passed'] }); + const { deps, stderr } = makeDeps(fetchImpl); + const report = (await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + )) as FlakyReport; + + expect(stderr.some(l => l.includes('[advisory]'))).toBe(false); + expect(report.advisories).toEqual([]); + }); +}); diff --git a/src/commands/test.rerun.closure-fanout.spec.ts b/src/commands/test.rerun.closure-fanout.spec.ts new file mode 100644 index 0000000..cf7abe0 --- /dev/null +++ b/src/commands/test.rerun.closure-fanout.spec.ts @@ -0,0 +1,257 @@ +/** + * Closure fan-out partial-results tests for `test rerun --wait` (BE). + * + * Split out of `test.rerun.spec.ts` (which is already the largest rerun spec): + * that file sits at the Windows-CI vitest reporter-RPC threshold, and growing + * it tips the `onTaskUpdate` worker call into a timeout. These focused cases + * live in their own small file so the big suite stays under that threshold. + * + * Behavior under test: one closure member's non-timeout poll error must NOT + * reject the whole fan-out — siblings survive and the partial still prints. + * The errored member is collected in `closureFailures[]` tagged `unobserved`, + * which flips `--wait` to exit 7 (its verdict was never confirmed). The named + * test's own error is re-thrown after the payload prints (real exit code + * preserved). All HTTP is mocked; the polling loop's sleep is injected via + * `TestDeps.sleep`. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError } from '../lib/errors.js'; +import type { RunResponse, RerunResponse } from '../lib/runs.types.js'; +import { runTestRerun } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers (self-contained copies of the minimal set from test.rerun.spec.ts) +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13503', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-dev459-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +const instantSleep = () => Promise.resolve(); + +const BE_TEST = { + id: 'test_be_consumer_01', + projectId: 'project_abc', + name: 'BE consumer test', + type: 'backend' as const, + createdFrom: 'portal' as const, + status: 'passed' as const, + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', +}; + +function makeBeRerunResp(): RerunResponse { + return { + runId: 'run_rerun_be_named', + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + closure: { + members: [ + { testId: 'test_be_consumer_01', runId: 'run_rerun_be_named', role: 'selected' }, + { testId: 'test_be_producer_01', runId: 'run_rerun_be_producer', role: 'producer' }, + ], + addedProducers: ['test_be_producer_01'], + addedTeardowns: [], + clearedCaptured: 0, + }, + }; +} + +function makeTerminalRun( + runId: string, + status: 'passed' | 'failed' | 'blocked' = 'passed', +): RunResponse { + return { + runId, + testId: 'test_be_consumer_01', + projectId: 'project_abc', + userId: 'user_1', + status, + source: 'cli', + createdAt: '2026-06-03T10:00:00.000Z', + startedAt: '2026-06-03T10:00:01.000Z', + finishedAt: '2026-06-03T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'rerun:prior_run_01', + failedStepIndex: status === 'passed' ? null : 2, + failureKind: status === 'passed' ? null : 'assertion', + error: null, + videoUrl: null, + stepSummary: { + total: 5, + completed: 5, + passedCount: status === 'passed' ? 5 : 4, + failedCount: status === 'passed' ? 0 : 1, + }, + }; +} + +function errorBody( + code: string, + details: Record = {}, +): { status: number; body: unknown } { + const statusMap: Record = { + NOT_FOUND: 404, + VALIDATION_ERROR: 400, + CONFLICT: 409, + RATE_LIMITED: 429, + INTERNAL: 500, + UNAVAILABLE: 503, + }; + return { + status: statusMap[code] ?? 400, + body: { + error: { + code, + message: `Error: ${code}`, + nextAction: 'do something', + requestId: 'req_test', + details, + }, + }, + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe('BE closure fan-out: a member poll error is classified (survives fan-out), flips exit 7', () => { + const baseOpts = { + all: false as const, + wait: true as const, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json' as const, + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }; + + it('non-named member poll error → siblings survive + partial printed, then --wait exits 7 (unobserved)', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + const namedRun = makeTerminalRun('run_rerun_be_named', 'passed'); + namedRun.testId = 'test_be_consumer_01'; + const stderrLines: string[] = []; + const printed: unknown[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) return { body: rerunResp }; + if (url.includes('/tests/test_be_consumer_01') || url.includes('/tests/test_be_producer_01')) + return { body: BE_TEST }; + if (url.includes('/runs/run_rerun_be_named')) return { body: namedRun }; + // Producer member poll fails for the whole window — the fan-out still + // completes and prints the partial, but the producer's verdict was never + // observed, so --wait must exit 7 (not silently succeed as exit 0). + return errorBody('UNAVAILABLE', { reason: 'upstream' }); + }); + + const err = await runTestRerun( + { ...baseOpts, testIds: ['test_be_consumer_01'] }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + stdout: line => printed.push(JSON.parse(line)), + }, + ).catch(e => e); + + // Unobserved dependency → exit 7, thrown AFTER the partial was printed. + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('UNSUPPORTED'); + expect((err as ApiError).exitCode).toBe(7); + + const json = printed[0] as { + namedStatus?: string; + closureFailures?: Array<{ testId: string; status: string; unobserved?: boolean }>; + }; + // The named verdict survived (not discarded by the sibling error). + expect(json.namedStatus).toBe('passed'); + // The errored member is surfaced in closureFailures[], tagged unobserved and + // carrying its diagnostic code. + expect(Array.isArray(json.closureFailures)).toBe(true); + const producerFailure = json.closureFailures!.find(f => f.testId === 'test_be_producer_01'); + expect(producerFailure?.status).toBe('UNAVAILABLE'); + expect(producerFailure?.unobserved).toBe(true); + expect(stderrLines.some(l => l.includes('closure member') && l.includes('UNAVAILABLE'))).toBe( + true, + ); + }); + + it('named member poll throws → real error/exit preserved, partial stdout still written', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + const producerRun = makeTerminalRun('run_rerun_be_producer', 'passed'); + producerRun.testId = 'test_be_producer_01'; + const stdoutLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) return { body: rerunResp }; + if (url.includes('/tests/test_be_consumer_01') || url.includes('/tests/test_be_producer_01')) + return { body: BE_TEST }; + if (url.includes('/runs/run_rerun_be_producer')) return { body: producerRun }; + // Named member poll fails. + return errorBody('NOT_FOUND', { reason: 'not_found' }); + }); + + const err = await runTestRerun( + { ...baseOpts, testIds: ['test_be_consumer_01'] }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: () => undefined, + stdout: line => stdoutLines.push(line), + }, + ).catch(e => e); + + // The named test's real error is surfaced (not masked as a timeout)... + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('NOT_FOUND'); + expect((err as ApiError).exitCode).toBe(4); + // ...and a parseable partial was still written before the re-throw. + expect(stdoutLines.length).toBeGreaterThan(0); + }); +}); diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 3797f14..382c414 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -662,6 +662,93 @@ describe('R-FE1: FE rerun -- wait (replay, exit 0 on passed)', () => { }); }); +// --------------------------------------------------------------------------- +// RATE_LIMITED during single (FE / BE-without-closure) rerun --wait polling — +// partial stdout + honest hint, exit 11 kept (never reclassified to 7). The +// 429 must be a real HTTP response (not thrown directly) so it exercises +// http.ts's own RATE_LIMITED retry-then-throw path. +// --------------------------------------------------------------------------- + +describe('single rerun --wait: RATE_LIMITED writes partial stdout, keeps exit 11', () => { + it('exit 11 (NOT reclassified to 7) AND stdout contains {runId, status:"running"}', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/run_rerun_fe_001')) { + // retry-after: 0 keeps http.ts's internal retry-then-throw budget fast. + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: too many requests from this IP.', + nextAction: '', + requestId: 'req_rl_rerun_test', + details: {}, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '0' } }, + ); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + }, + ).catch(e => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('RATE_LIMITED'); + expect((err as ApiError).exitCode).toBe(11); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(stdoutJson.runId).toBe(rerunResp.runId); + expect(stdoutJson.status).toBe('running'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain(rerunResp.runId); + expect(stderrBlock).toContain('test wait'); + expect(stderrBlock).toContain('test cancel'); + expect(stderrBlock).toContain('Rate limited'); + }); +}); + // --------------------------------------------------------------------------- // R-FE2/R-FE4: --auto-heal flag // --------------------------------------------------------------------------- @@ -816,9 +903,9 @@ describe('R-FE4: server unexpectedly echoes autoHeal:false — prints "not appli }); }); -// R-FE5: --no-auto-heal explicit opt-out → body sends no autoHeal field +// R-FE5: --no-auto-heal explicit opt-out → body sends autoHeal:false explicitly describe('R-FE5: --no-auto-heal opt-out', () => { - it('does NOT send autoHeal in body when autoHeal:false; no advisory emitted', async () => { + it('sends autoHeal:false explicitly in body (not omitted); no auto-heal advisory emitted', async () => { const creds = makeCreds(); const rerunResp = makeFeRerunResp({ autoHeal: false }); // verbatim replay const stderrLines: string[] = []; @@ -856,8 +943,10 @@ describe('R-FE5: --no-auto-heal opt-out', () => { }, ); - // autoHeal must NOT be sent to server (effectiveAutoHeal is false) - expect((sentBody as { autoHeal?: boolean }).autoHeal).toBeUndefined(); + // autoHeal:false must be sent EXPLICITLY (not omitted) — an + // absent field defaults to heal-on server-side, which silently discards + // the user's --no-auto-heal opt-out. + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBe(false); // No advisory for a verbatim replay (server echoes false, opts.autoHeal is false) const advisory = stderrLines.find(l => l.includes('[advisory]')); @@ -865,6 +954,130 @@ describe('R-FE5: --no-auto-heal opt-out', () => { }); }); +// --------------------------------------------------------------------------- +// Single rerun renders server-side `advisories[]` +// --------------------------------------------------------------------------- + +describe('single rerun renders server advisories', () => { + const advisory = { + feature: 'autoHeal', + message: + 'The auto-heal opt-out was forwarded to the execution engine but is not yet enforced there.', + }; + + it('text mode: prints one [advisory] line per entry in rerunResp.advisories', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp({ autoHeal: false, advisories: [advisory] }); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'text', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines).toContain(`[advisory] ${advisory.message}`); + }); + + it('JSON mode: passes advisories through untouched on the printed response', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp({ autoHeal: false, advisories: [advisory] }); + const printed: unknown[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: line => printed.push(JSON.parse(line)) }, + ); + + const result = printed[0] as RerunResponse; + expect(result.advisories).toEqual([advisory]); + }); + + it('absent advisories: no [advisory] line, and the JSON field stays absent (byte-identical passthrough)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp({ autoHeal: false }); // no advisories field at all + const stderrLines: string[] = []; + const printed: unknown[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + expect(stderrLines.some(l => l.includes('[advisory]'))).toBe(false); + const result = printed[0] as RerunResponse; + expect(result.advisories).toBeUndefined(); + }); +}); + // --------------------------------------------------------------------------- // R-BE1: BE rerun with closure // --------------------------------------------------------------------------- @@ -1119,22 +1332,371 @@ describe('R-BE2: --skip-dependencies', () => { let sentBody: unknown; const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: true, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + }, + ); + + expect((sentBody as { skipDependencies?: boolean }).skipDependencies).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// R-BE3: auto-heal on BE test — suppressed or warned depending on explicit flag +// --------------------------------------------------------------------------- + +describe('R-BE3: auto-heal on BE test — default-on suppresses warning', () => { + it('auto-heal defaults true; BE type suppresses warning (autoHealExplicit:false); autoHeal:false sent explicitly', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp({ autoHeal: false }); + const stderrLines: string[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { + return { body: BE_TEST }; + } + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + // autoHeal:true is the default (user did NOT pass --no-auto-heal) + autoHeal: true, + // autoHealExplicit:false means we suppress the BE "ignoring" warning + // to avoid noise on every default BE rerun. + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // With autoHealExplicit:false, the "ignoring auto-heal" advisory must NOT be emitted + const warning = stderrLines.find( + l => l.includes('auto-heal applies to frontend tests only') || l.includes('ignoring'), + ); + expect(warning).toBeUndefined(); + + // autoHeal is sent EXPLICITLY as false (effectiveAutoHeal is + // false for BE) — not omitted. + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBe(false); + }); + + it('auto-heal explicitly requested (autoHealExplicit:true); BE type emits warning', async () => { + // This covers a hypothetical future scenario where a user can explicitly + // pass --auto-heal. Since there's no such flag currently, autoHealExplicit + // can be set to true only by callers that inject it directly (e.g. tests or + // future flag additions). + const creds = makeCreds(); + const rerunResp = makeBeRerunResp({ autoHeal: false }); + const stderrLines: string[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { + return { body: BE_TEST }; + } + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: true, + autoHealExplicit: true, // user explicitly requested --auto-heal + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // With autoHealExplicit:true, the warning IS emitted + const warning = stderrLines.find( + l => l.includes('auto-heal applies to frontend tests only') || l.includes('ignoring'), + ); + expect(warning).toBeDefined(); + + // autoHeal is sent EXPLICITLY as false (effectiveAutoHeal is + // false for BE) — not omitted. + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// FIX 2 — BE rerun: no "not applied" advisory (effectiveAutoHeal fix) +// --------------------------------------------------------------------------- +// D2-FIX2: Before the fix, `opts.autoHeal && !rerunResp.autoHeal` fired on +// every BE rerun because opts.autoHeal is default-on true but the server always +// echoes autoHeal:false for BE tests. The fix changes the guard to +// `effectiveAutoHeal && !rerunResp.autoHeal` so BE reruns (effectiveAutoHeal=false) +// never trigger the advisory. +describe('[fix-2] BE rerun: spurious "not applied" advisory is suppressed', () => { + it('BE rerun with default-on autoHeal does NOT print "not applied" advisory', async () => { + const creds = makeCreds(); + // Server echoes autoHeal:false for BE (expected) — before the fix, this + // would trigger the defensive advisory every time. + const rerunResp = makeBeRerunResp({ autoHeal: false }); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { + return { body: BE_TEST }; + } + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: true, // default-on (user did NOT pass --no-auto-heal) + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + // The "not applied" advisory MUST NOT fire for BE reruns (FIX 2). + // effectiveAutoHeal is false for BE, so the guard is: false && !false → false. + const notAppliedLine = stderrLines.find( + l => l.includes('not applied') || l.includes('was not applied'), + ); + expect(notAppliedLine).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// "not applied" advisory wording — points at `testsprite usage`, never a +// hardcoded personal billing URL. The CLI has no per-request org context +// here (no backend nextAction feeds this client-side advisory, and a +// personal-vs-org-bound key can't be told apart at this point), so it must +// not assert "check your (personal) balance at " — `usage` +// already renders whichever wallet actually governs this key. +// --------------------------------------------------------------------------- +describe('"not applied" auto-heal advisory wording (FE, server rejects the heal request)', () => { + it('points at `testsprite usage`, not a hardcoded billing URL', async () => { + const creds = makeCreds(); + // Server echoes autoHeal:false despite the CLI sending true — the + // defensive "not applied" branch. + const rerunResp = makeFeRerunResp({ autoHeal: false }); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01') && !url.includes('/runs/rerun')) { + return { body: FE_TEST }; + } + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: true, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stderr: line => stderrLines.push(line), + }, + ); + + const notAppliedLine = stderrLines.find( + l => l.includes('not applied') || l.includes('was not applied'), + ); + expect(notAppliedLine).toBeDefined(); + expect(notAppliedLine).toContain('testsprite usage'); + expect(notAppliedLine).not.toContain('dashboard/settings/billing'); + }); +}); + +// --------------------------------------------------------------------------- +// R-BAT: Batch rerun +// --------------------------------------------------------------------------- + +describe('R-BAT: batch rerun (multi-id, no --wait)', () => { + it('sends testIds to POST /tests/batch/rerun and prints accepted', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const printed: unknown[] = []; + let sentBody: unknown; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + sentBody = init.body ? JSON.parse(init.body as string) : null; + return { status: 202, body: batchResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => printed.push(JSON.parse(line)), + }, + ); + + expect((sentBody as { testIds: string[] }).testIds).toEqual(['test_1', 'test_2']); + const result = printed[0] as BatchRerunResponse; + expect(result.accepted).toHaveLength(2); + expect(result.accepted[0]!.runId).toBe('run_b1'); + expect(result.accepted[1]!.runId).toBe('run_b2'); + }); +}); + +// --------------------------------------------------------------------------- +// Batch rerun (initial dispatch + deferred-retry) always sends an +// explicit autoHeal boolean, including an explicit `false` opt-out. +// --------------------------------------------------------------------------- + +describe('batch rerun always sends an explicit autoHeal boolean', () => { + it('initial dispatch sends autoHeal:true explicitly when auto-heal is default-on', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let sentBody: unknown; + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { sentBody = init.body ? JSON.parse(init.body as string) : null; - return { body: rerunResp }; + return { status: 202, body: batchResp }; } return errorBody('NOT_FOUND'); }); await runTestRerun( { - testIds: ['test_be_consumer_01'], + // Two ids so the batch path is exercised (a single id routes through + // the single-rerun code path instead of `POST /tests/batch/rerun`). + testIds: ['test_1', 'test_2'], all: false, wait: false, timeoutSeconds: 600, - autoHeal: false, + autoHeal: true, autoHealExplicit: false, - skipDependencies: true, + skipDependencies: false, maxConcurrency: 10, output: 'json', profile: 'default', @@ -1142,49 +1704,39 @@ describe('R-BE2: --skip-dependencies', () => { debug: false, verbose: false, }, - { - ...creds, - sleep: instantSleep, - fetchImpl, - }, + { ...creds, sleep: instantSleep, fetchImpl }, ); - expect((sentBody as { skipDependencies?: boolean }).skipDependencies).toBe(true); + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBe(true); }); -}); - -// --------------------------------------------------------------------------- -// R-BE3: auto-heal on BE test — suppressed or warned depending on explicit flag -// --------------------------------------------------------------------------- -describe('R-BE3: auto-heal on BE test — default-on suppresses warning', () => { - it('auto-heal defaults true; BE type suppresses warning (autoHealExplicit:false); autoHeal NOT sent', async () => { + it('initial dispatch sends autoHeal:false explicitly (not omitted) when --no-auto-heal is passed', async () => { const creds = makeCreds(); - const rerunResp = makeBeRerunResp({ autoHeal: false }); - const stderrLines: string[] = []; + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; let sentBody: unknown; - const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { - return { body: BE_TEST }; - } - if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + if (url.includes('/tests/batch/rerun')) { sentBody = init.body ? JSON.parse(init.body as string) : null; - return { body: rerunResp }; + return { status: 202, body: batchResp }; } return errorBody('NOT_FOUND'); }); await runTestRerun( { - testIds: ['test_be_consumer_01'], + testIds: ['test_1', 'test_2'], all: false, wait: false, timeoutSeconds: 600, - // autoHeal:true is the default (user did NOT pass --no-auto-heal) - autoHeal: true, - // autoHealExplicit:false means we suppress the BE "ignoring" warning - // to avoid noise on every default BE rerun. + autoHeal: false, autoHealExplicit: false, skipDependencies: false, maxConcurrency: 10, @@ -1194,53 +1746,54 @@ describe('R-BE3: auto-heal on BE test — default-on suppresses warning', () => debug: false, verbose: false, }, - { - ...creds, - sleep: instantSleep, - fetchImpl, - stderr: line => stderrLines.push(line), - }, - ); - - // With autoHealExplicit:false, the "ignoring auto-heal" advisory must NOT be emitted - const warning = stderrLines.find( - l => l.includes('auto-heal applies to frontend tests only') || l.includes('ignoring'), + { ...creds, sleep: instantSleep, fetchImpl }, ); - expect(warning).toBeUndefined(); - // autoHeal must NOT be sent to server (effectiveAutoHeal is false for BE) - expect((sentBody as { autoHeal?: boolean }).autoHeal).toBeUndefined(); + expect((sentBody as { autoHeal?: boolean }).autoHeal).toBe(false); }); - it('auto-heal explicitly requested (autoHealExplicit:true); BE type emits warning', async () => { - // This covers a hypothetical future scenario where a user can explicitly - // pass --auto-heal. Since there's no such flag currently, autoHealExplicit - // can be set to true only by callers that inject it directly (e.g. tests or - // future flag additions). + it('the D3 deferred-retry dispatch also re-sends autoHeal:false explicitly', async () => { const creds = makeCreds(); - const rerunResp = makeBeRerunResp({ autoHeal: false }); - const stderrLines: string[] = []; - let sentBody: unknown; + // test_2 is accepted immediately; test_1 is rate-deferred on the initial + // dispatch and only accepted on the D3 retry. + const initialBatchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }], + deferred: [{ testId: 'test_1', reason: 'rate_limited' }], + conflicts: [], + closure: { byProject: [] }, + }; + const retryBatchResp: BatchRerunResponse = { + accepted: [{ testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:05.000Z' }], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const sentBodies: unknown[] = []; + let batchCallCount = 0; + const run1 = makeTerminalRun('run_b1', 'passed'); + run1.testId = 'test_1'; + const run2 = makeTerminalRun('run_b2', 'passed'); + run2.testId = 'test_2'; const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { - return { body: BE_TEST }; - } - if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { - sentBody = init.body ? JSON.parse(init.body as string) : null; - return { body: rerunResp }; + if (url.includes('/tests/batch/rerun')) { + batchCallCount++; + sentBodies.push(init.body ? JSON.parse(init.body as string) : null); + return { status: 202, body: batchCallCount === 1 ? initialBatchResp : retryBatchResp }; } + if (url.includes('/runs/run_b1')) return { body: run1 }; + if (url.includes('/runs/run_b2')) return { body: run2 }; return errorBody('NOT_FOUND'); }); await runTestRerun( { - testIds: ['test_be_consumer_01'], + testIds: ['test_1', 'test_2'], all: false, - wait: false, + wait: true, timeoutSeconds: 600, - autoHeal: true, - autoHealExplicit: true, // user explicitly requested --auto-heal + autoHeal: false, + autoHealExplicit: false, skipDependencies: false, maxConcurrency: 10, output: 'json', @@ -1249,58 +1802,54 @@ describe('R-BE3: auto-heal on BE test — default-on suppresses warning', () => debug: false, verbose: false, }, - { - ...creds, - sleep: instantSleep, - fetchImpl, - stderr: line => stderrLines.push(line), - }, - ); - - // With autoHealExplicit:true, the warning IS emitted - const warning = stderrLines.find( - l => l.includes('auto-heal applies to frontend tests only') || l.includes('ignoring'), + { ...creds, sleep: instantSleep, fetchImpl }, ); - expect(warning).toBeDefined(); - // autoHeal must NOT be sent to server (effectiveAutoHeal is false for BE) - expect((sentBody as { autoHeal?: boolean }).autoHeal).toBeUndefined(); + // 1 initial dispatch + 1 D3 retry that finally accepts the deferred test. + expect(batchCallCount).toBe(2); + expect((sentBodies[0] as { autoHeal?: boolean }).autoHeal).toBe(false); + expect((sentBodies[1] as { autoHeal?: boolean }).autoHeal).toBe(false); }); }); // --------------------------------------------------------------------------- -// FIX 2 — BE rerun: no "not applied" advisory (effectiveAutoHeal fix) +// Batch rerun renders server-side `advisories[]` // --------------------------------------------------------------------------- -// D2-FIX2: Before the fix, `opts.autoHeal && !rerunResp.autoHeal` fired on -// every BE rerun because opts.autoHeal is default-on true but the server always -// echoes autoHeal:false for BE tests. The fix changes the guard to -// `effectiveAutoHeal && !rerunResp.autoHeal` so BE reruns (effectiveAutoHeal=false) -// never trigger the advisory. -describe('[fix-2] BE rerun: spurious "not applied" advisory is suppressed', () => { - it('BE rerun with default-on autoHeal does NOT print "not applied" advisory', async () => { + +describe('batch rerun renders server advisories', () => { + const advisory = { feature: 'autoHeal', message: 'not yet enforced by the execution engine' }; + + it('prints the advisory once to stderr and passes it through in the JSON output', async () => { const creds = makeCreds(); - // Server echoes autoHeal:false for BE (expected) — before the fix, this - // would trigger the defensive advisory every time. - const rerunResp = makeBeRerunResp({ autoHeal: false }); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + advisories: [advisory], + }; const stderrLines: string[] = []; + const printed: unknown[] = []; const fetchImpl = makeFetch(url => { - if (url.includes('/tests/test_be_consumer_01') && !url.includes('/runs/rerun')) { - return { body: BE_TEST }; - } - if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { - return { body: rerunResp }; + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; } return errorBody('NOT_FOUND'); }); await runTestRerun( { - testIds: ['test_be_consumer_01'], + // Two ids so the batch path is exercised (a single id routes through + // the single-rerun code path instead of `POST /tests/batch/rerun`). + testIds: ['test_1', 'test_2'], all: false, wait: false, timeoutSeconds: 600, - autoHeal: true, // default-on (user did NOT pass --no-auto-heal) + autoHeal: false, autoHealExplicit: false, skipDependencies: false, maxConcurrency: 10, @@ -1315,24 +1864,19 @@ describe('[fix-2] BE rerun: spurious "not applied" advisory is suppressed', () = sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line), + stdout: line => printed.push(JSON.parse(line)), }, ); - // The "not applied" advisory MUST NOT fire for BE reruns (FIX 2). - // effectiveAutoHeal is false for BE, so the guard is: false && !false → false. - const notAppliedLine = stderrLines.find( - l => l.includes('not applied') || l.includes('was not applied'), - ); - expect(notAppliedLine).toBeUndefined(); - }); -}); + // Exactly one line — not once per accepted test in the batch. + const advisoryLines = stderrLines.filter(l => l === `[advisory] ${advisory.message}`); + expect(advisoryLines).toHaveLength(1); -// --------------------------------------------------------------------------- -// R-BAT: Batch rerun -// --------------------------------------------------------------------------- + const result = printed[0] as BatchRerunResponse; + expect(result.advisories).toEqual([advisory]); + }); -describe('R-BAT: batch rerun (multi-id, no --wait)', () => { - it('sends testIds to POST /tests/batch/rerun and prints accepted', async () => { + it('absent advisories field: no [advisory] line is printed', async () => { const creds = makeCreds(); const batchResp: BatchRerunResponse = { accepted: [ @@ -1343,12 +1887,11 @@ describe('R-BAT: batch rerun (multi-id, no --wait)', () => { conflicts: [], closure: { byProject: [] }, }; + const stderrLines: string[] = []; const printed: unknown[] = []; - let sentBody: unknown; - const fetchImpl = makeFetch((url, init) => { + const fetchImpl = makeFetch(url => { if (url.includes('/tests/batch/rerun')) { - sentBody = init.body ? JSON.parse(init.body as string) : null; return { status: 202, body: batchResp }; } return errorBody('NOT_FOUND'); @@ -1374,15 +1917,17 @@ describe('R-BAT: batch rerun (multi-id, no --wait)', () => { ...creds, sleep: instantSleep, fetchImpl, + stderr: line => stderrLines.push(line), stdout: line => printed.push(JSON.parse(line)), }, ); - expect((sentBody as { testIds: string[] }).testIds).toEqual(['test_1', 'test_2']); + expect(stderrLines.some(l => l.includes('[advisory]'))).toBe(false); + // The CLI aggregates advisories client-side across chunked dispatch + // requests (same treatment as `notFound`), so an absent server field + // normalizes to an empty array here rather than staying undefined. const result = printed[0] as BatchRerunResponse; - expect(result.accepted).toHaveLength(2); - expect(result.accepted[0]!.runId).toBe('run_b1'); - expect(result.accepted[1]!.runId).toBe('run_b2'); + expect(result.advisories).toEqual([]); }); }); @@ -3743,6 +4288,112 @@ describe('[finding-3] BE closure fan-out: RequestTimeoutError emits partial stdo }); }); +// --------------------------------------------------------------------------- +// RATE_LIMITED in the BE closure fan-out: same partial-envelope contract as +// the RequestTimeoutError test above, but exit code must stay 11 (never +// reclassified to 7). `pollMember` only swallows `TimeoutError` into a null +// return — every other error (including a RATE_LIMITED ApiError) propagates +// through `.catch(reject)` exactly like RequestTimeoutError, so the outer +// `catch (fanOutErr)` must list every dispatched closure-member runId. +// --------------------------------------------------------------------------- + +describe('[RATE_LIMITED] BE closure fan-out: emits partial stdout for ALL runIds, keeps exit 11', () => { + it('RATE_LIMITED in a closure member poll → partial stdout with all runIds + exit 11 (not 7)', async () => { + const creds = makeCreds(); + const rerunResp = makeBeRerunResp(); + const namedRunId = rerunResp.runId; // 'run_rerun_be_named' + const producerRunId = rerunResp.closure!.members.find(m => m.role === 'producer')!.runId; + + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/test_be_consumer_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if ( + url.includes('/tests/test_be_consumer_01') || + url.includes('/tests/test_be_producer_01') + ) { + return new Response(JSON.stringify(BE_TEST), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // Every closure-member poll gets rate-limited (retry-after: 0 keeps + // http.ts's internal retry-then-throw budget fast). + if (url.includes('/runs/')) { + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: too many requests from this IP.', + nextAction: '', + requestId: 'req_rl_closure_test', + details: {}, + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '0' } }, + ); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_be_consumer_01'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + }, + ).catch(e => e); + + // Must stay exit 11 (RATE_LIMITED) — NOT reclassified to 7. + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('RATE_LIMITED'); + expect((err as ApiError).exitCode).toBe(11); + + // Stdout must list every dispatched closure-member runId. + expect(stdoutLines.length).toBeGreaterThan(0); + const stdoutBlock = stdoutLines.join('\n'); + expect(stdoutBlock).toContain(namedRunId); + expect(stdoutBlock).toContain(producerRunId); + + // Stderr must include re-attach hints for every closure-member runId. + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain(namedRunId); + expect(stderrBlock).toContain(producerRunId); + expect(stderrBlock).toContain('test wait'); + expect(stderrBlock).toContain('test cancel'); + expect(stderrBlock).toContain('Rate limited'); + }); +}); + // --------------------------------------------------------------------------- // G1d — split teardowns from producers in the rerun stderr summary // --------------------------------------------------------------------------- diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index 87df3e0..d5b0cee 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -23,7 +23,7 @@ import { describe, expect, it } from 'vitest'; import { ApiError } from '../lib/errors.js'; import type { ListRunsResponse, RunHistoryItem } from '../lib/runs.types.js'; import type { CliLatestResult } from './test.js'; -import { runResultHistory, runResult, parseDuration } from './test.js'; +import { runResultHistory, runResult, parseDuration, createTestCommand } from './test.js'; // --------------------------------------------------------------------------- // Helpers @@ -415,6 +415,76 @@ describe('runResultHistory — text mode', () => { expect(output).toMatch(/\byes\b/); }); + // --rerun / --no-rerun client-side filter (asserted in JSON mode for exactness) + const mixedRerunFetch = () => + makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ runId: 'run_fresh', isRerun: false, createdFrom: null }), + makeHistoryItem({ runId: 'run_rerun', isRerun: true, createdFrom: 'rerun:prior' }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + it('--rerun keeps only reruns in the JSON runs array', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + rerun: true, + }, + { credentialsPath, fetchImpl: mixedRerunFetch(), stdout: line => lines.push(line) }, + ); + const parsed = JSON.parse(lines.join('')) as { runs: RunHistoryItem[] }; + expect(parsed.runs.map(r => r.runId)).toEqual(['run_rerun']); + }); + + it('--no-rerun keeps only fresh runs in the JSON runs array', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + rerun: false, + }, + { credentialsPath, fetchImpl: mixedRerunFetch(), stdout: line => lines.push(line) }, + ); + const parsed = JSON.parse(lines.join('')) as { runs: RunHistoryItem[] }; + expect(parsed.runs.map(r => r.runId)).toEqual(['run_fresh']); + }); + + it('no rerun flag keeps every run (no filter)', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl: mixedRerunFetch(), stdout: line => lines.push(line) }, + ); + const parsed = JSON.parse(lines.join('')) as { runs: RunHistoryItem[] }; + expect(parsed.runs.map(r => r.runId)).toEqual(['run_fresh', 'run_rerun']); + }); + it('renders DURATION as "NmNs" when startedAt and finishedAt are present', async () => { const { credentialsPath } = makeCreds(); const lines: string[] = []; @@ -1465,3 +1535,41 @@ describe('runResultHistory — G1b targetUrl in history table (text mode)', () = expect(firstRun?.targetUrlSource).toBe('run'); }); }); + +// --------------------------------------------------------------------------- +// --rerun / --no-rerun parsing through Commander (the tri-state hop) +// --------------------------------------------------------------------------- + +describe('createTestCommand — --rerun / --no-rerun parsing', () => { + // The runResultHistory tests above call the function directly, so they never + // exercise Commander's --rerun/--no-rerun parsing. This pins the tri-state: + // undefined (no filter) | true | false. A silent default to true would filter + // every `test result --history` to reruns only, unnoticed. + async function parsedRerun(extraArgs: string[]): Promise { + const { credentialsPath } = makeCreds(); + // Benign history response so parseAsync's action completes without a real + // network call — the assertion reads the parsed opts, not the output. + const fetchImpl = makeFetch(() => ({ body: makeHistoryResp([]) })); + const test = createTestCommand({ + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + }); + await test.parseAsync(['result', 'test_abc', '--history', ...extraArgs], { from: 'user' }); + const resultCmd = test.commands.find(c => c.name() === 'result'); + return (resultCmd?.opts() as { rerun?: boolean }).rerun; + } + + it('no flag → rerun is undefined (no filter)', async () => { + expect(await parsedRerun([])).toBeUndefined(); + }); + + it('--rerun → rerun is true', async () => { + expect(await parsedRerun(['--rerun'])).toBe(true); + }); + + it('--no-rerun → rerun is false', async () => { + expect(await parsedRerun(['--no-rerun'])).toBe(false); + }); +}); diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 6c8be5f..e0bcbe5 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -2300,6 +2300,180 @@ describe('[finding-C] runTestRun --wait RequestTimeoutError — text mode render }); }); +// --------------------------------------------------------------------------- +// RATE_LIMITED during --wait polling — partial stdout + honest hint, exit 11 +// kept (never reclassified to 7). Mirrors the RequestTimeoutError coverage +// above; the 429 must be returned as a real HTTP response (not thrown +// directly) so it exercises http.ts's own RATE_LIMITED retry-then-throw path. +// --------------------------------------------------------------------------- + +function rateLimitedResponse(retryAfterSeconds?: number): Response { + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: too many requests from this IP.', + nextAction: '', + requestId: 'req_rl_test', + details: {}, + }, + }), + { + status: 429, + headers: { + 'content-type': 'application/json', + ...(retryAfterSeconds !== undefined ? { 'retry-after': String(retryAfterSeconds) } : {}), + }, + }, + ); +} + +describe('runTestRun --wait: RATE_LIMITED writes partial JSON to stdout, keeps exit 11', () => { + it('exit 11 (NOT reclassified to 7) AND stdout contains {runId, status:"running"} when poll exhausts RATE_LIMITED retries', async () => { + const { credentialsPath } = makeCreds(); + let callCount = 0; + const fetchImpl: typeof globalThis.fetch = async () => { + callCount += 1; + if (callCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // retry-after: 0 keeps http.ts's internal retry-then-throw budget fast. + return rateLimitedResponse(0); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(e => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('RATE_LIMITED'); + expect((err as ApiError).exitCode).toBe(11); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + targetUrl: string; + }; + expect(stdoutJson.runId).toBe(TRIGGER_RESP.runId); + expect(stdoutJson.status).toBe('running'); + expect(stdoutJson.targetUrl).toBe(TRIGGER_RESP.targetUrl); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain(TRIGGER_RESP.runId); + expect(stderrBlock).toContain('test wait'); + expect(stderrBlock).toContain('test cancel'); + expect(stderrBlock).toContain('Rate limited'); + }); + + it('text mode: renders human-readable partial (not raw JSON), still exit 11', async () => { + const { credentialsPath } = makeCreds(); + let callCount = 0; + const fetchImpl: typeof globalThis.fetch = async () => { + callCount += 1; + if (callCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return rateLimitedResponse(0); + }; + + const stdoutLines: string[] = []; + + const err = await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + + expect((err as ApiError).exitCode).toBe(11); + const stdoutBlock = stdoutLines.join('\n'); + expect(stdoutBlock).toContain('runId'); + expect(stdoutBlock).toContain('running'); + expect(stdoutBlock).not.toMatch(/^\{/); + }); + + it('honors Retry-After in the stderr hint when present', async () => { + const { credentialsPath } = makeCreds(); + let callCount = 0; + const fetchImpl: typeof globalThis.fetch = async () => { + callCount += 1; + if (callCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + // A non-zero Retry-After still keeps the internal retry-then-throw + // budget fast (only 2 real sleeps of retryAfterSeconds*1000ms), so keep + // this small. + return rateLimitedResponse(1); + }; + + const stderrLines: string[] = []; + + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(e => e); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toMatch(/retry after ~\d+s/); + }); +}); + // --------------------------------------------------------------------------- // Finding D (codex round-2) — 409 conflict auto-resume without --target-url // → timeout partial carries the real in-flight targetUrl (not '') @@ -2638,12 +2812,19 @@ describe('runTestRunAll — batch fresh run', () => { const { createTestCommand } = await import('./test.js'); const test = createTestCommand(); disableExits(test); - await expect( - test.parseAsync( - ['run', '--all', '--project', 'proj_1', '--target-url', 'https://example.com'], - { from: 'user' }, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + const rejection = (await test + .parseAsync(['run', '--all', '--project', 'proj_1', '--target-url', 'https://example.com'], { + from: 'user', + }) + .catch((error: unknown) => error)) as ApiError; + expect(rejection).toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + // The rejection explains why + how to fix it, without a + // cosmetic doubled period at the end (the reason clause itself already + // ends with "Remove --target-url.", and the `nextAction` template used + // to blindly append a second one). + expect(rejection.nextAction).toContain('Remove --target-url.'); + expect(rejection.nextAction.endsWith('..')).toBe(false); + expect(rejection.nextAction.endsWith('.')).toBe(true); }); it(' --filter (without --all) → exit 5 (filter is --all-only)', async () => { diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 1922448..db4b169 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -10,8 +10,9 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { Command } from 'commander'; -import { ApiError } from '../lib/errors.js'; +import { ApiError, InterruptError } from '../lib/errors.js'; import { GLOBAL_OPTS_HINT } from '../lib/output.js'; +import { ShutdownController } from '../lib/interrupt.js'; import { type CliFailureContext, type CliLatestResult, @@ -21,6 +22,9 @@ import { type TestDeps, createTestCommand, isPresignedCodeUrl, + PLAN_SCHEMA_URL, + PLAN_TEMPLATE_TEXT, + PLAN_TEMPLATE_WITH_SCHEMA, runCodeGet, runCodePut, runCreate, @@ -35,6 +39,7 @@ import { runList, runOpen, runPlanPut, + runPlanTemplate, runResult, runScaffold, runSteps, @@ -202,6 +207,7 @@ describe('createTestCommand — surface', () => { it('result exposes --include-analysis (M2.1) + M3.4 piece-5 --history flags', () => { // M2.1 piece 3 adds `--include-analysis` to `test result`. // M3.4 piece 5 adds `--history`, `--source`, `--since`, `--page-size`, `--cursor`. + // The rerun history filter adds `--rerun` / `--no-rerun`. // Issue #165 adds text-table shaping via `--columns` and `--no-header`. // Pinning the surface so a future flag-consolidation sweep keeps every // option intentional. Back-compat: bare `test result ` (no --history) @@ -216,6 +222,8 @@ describe('createTestCommand — surface', () => { '--since', '--page-size', '--cursor', + '--rerun', + '--no-rerun', '--columns', '--no-header', ]); @@ -2594,6 +2602,69 @@ describe('runScaffold', () => { }); }); +// --------------------------------------------------------------------------- +// `test create --plan-template`. Pure-local: no network, no +// credentials required (unlike everything above, no makeCreds()/fetchImpl). +// --------------------------------------------------------------------------- +describe('runPlanTemplate', () => { + it('prints PLAN_TEMPLATE_TEXT verbatim to stdout (text mode) and makes no fetch calls', async () => { + const out: string[] = []; + let fetchCalled = false; + const result = await runPlanTemplate( + { profile: 'default', output: 'text', debug: false }, + { + stdout: line => out.push(line), + stderr: () => undefined, + fetchImpl: (() => { + fetchCalled = true; + throw new Error('runPlanTemplate must never call fetch'); + }) as unknown as typeof globalThis.fetch, + }, + ); + expect(fetchCalled).toBe(false); + expect(out.join('\n')).toBe(PLAN_TEMPLATE_TEXT); + expect(result).toEqual(PLAN_TEMPLATE_WITH_SCHEMA); + }); + + it('prints byte-identical output in --output json mode (Output.print(JSON.stringify) matches PLAN_TEMPLATE_TEXT)', async () => { + const out: string[] = []; + await runPlanTemplate( + { profile: 'default', output: 'json', debug: false }, + { stdout: line => out.push(line), stderr: () => undefined }, + ); + expect(out.join('\n')).toBe(PLAN_TEMPLATE_TEXT); + }); + + it('the printed template round-trips through the real plan-from validator (assertPlanShape via runCreateFromPlan --dry-run)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-plan-template-')); + const planFile = join(dir, 'plan.json'); + writeFileSync(planFile, PLAN_TEMPLATE_TEXT, 'utf8'); + // --dry-run swaps in the canned dry-run fetch implementation regardless of + // any injected fetchImpl (client-factory.ts) and needs no credentials file + // — local validation still runs in full, which is exactly the property + // under test here (mirrors the existing "--dry-run does NOT emit + // dashboardUrl" convention elsewhere in this file). + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + dryRun: true, + endpointUrl: 'https://api.testsprite.com', + }, + { stdout: () => undefined, stderr: () => undefined }, + ), + ).resolves.toBeDefined(); + }); + + it('`$schema` is present and does not upset the validator (extra top-level key is allowed)', () => { + expect(PLAN_TEMPLATE_WITH_SCHEMA.$schema).toBe(PLAN_SCHEMA_URL); + expect(typeof PLAN_TEMPLATE_WITH_SCHEMA.$schema).toBe('string'); + }); +}); + describe('runOpen', () => { // The mock endpoint host has no portal mapping; the operator override is the // supported escape hatch and gives the tests a deterministic base. @@ -3295,10 +3366,16 @@ describe('runDiff', () => { expect(errs.join('\n')).toContain('different tests'); }); - it('--dry-run returns the canned sample fully offline (no credentials, no fetch)', async () => { - // Dry-run must not require credentials or hit the network — it returns a - // canned CliRunDiff so `--dry-run` shows the shape offline. - const diff = await runDiff( + it('--dry-run prints the canned sample fully offline (no credentials, no fetch) AND honors the exit-code contract', async () => { + // Dry-run must not require credentials or hit the network — it prints a + // canned CliRunDiff so `--dry-run` shows the shape offline. The + // canned sample has verdictChanged: true, so per the documented contract + // ("Exit 0 when verdicts match, 1 when they differ" — no dry-run + // exception) this MUST reject with exit 1, same as a real regressed + // pair. Before the fix, the early `return sample` bypassed the + // verdictChanged check entirely and `--dry-run` always exited 0. + const out: string[] = []; + const rejection = await runDiff( { profile: 'default', output: 'json', @@ -3307,8 +3384,15 @@ describe('runDiff', () => { runA: 'run_aaa', runB: 'run_bbb', }, - { stdout: () => undefined, stderr: () => undefined }, - ); + { stdout: line => out.push(line), stderr: () => undefined }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const diff = JSON.parse(out.join('')) as { + runA: { runId: string }; + runB: { runId: string }; + verdictChanged: boolean; + changedSteps: Array<{ stepIndex: number; statusA: string; statusB: string }>; + }; expect(diff.runA.runId).toBe('run_aaa'); expect(diff.runB.runId).toBe('run_bbb'); expect(diff.verdictChanged).toBe(true); @@ -3337,6 +3421,25 @@ describe('runLint', () => { name: 'Broken', planSteps: [{ type: 'hover', description: 'Bad step type' }], }); + // Regression fixture: mirrors the customer repro (Topify, + // 2026-07-10/11) exactly — a `name` type error, an invalid `priority` + // enum value, AND an invalid step `type`, all in the SAME file. Before the + // fix, `test lint` reported only `name`; fixing it revealed `priority`; + // fixing THAT revealed the step-type error — three fix-and-rerun cycles + // for one file. + const MULTI_PROBLEM_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 123, // wrong type — must be a string + priority: 'urgent', // not one of CLI_CREATE_PRIORITIES + planSteps: [{ type: 'hover', description: 'Bad step type' }], // not a valid step type + }); + const MULTI_PROBLEM_STEPS = JSON.stringify({ + planSteps: [ + { type: 'click', description: 'Click submit' }, // invalid step type + { type: 'action', description: 123 }, // wrong type — must be a string + ], + }); it('a directory with valid and invalid plans reports EVERY problem and exits 5', async () => { const dir = mkdtempSync(join(tmpdir(), 'cli-lint-')); @@ -3398,6 +3501,166 @@ describe('runLint', () => { runLint({ profile: 'default', output: 'json', debug: false }, { stdout: () => undefined }), ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + + // ---------- collect EVERY problem WITHIN a single file, not just + // the first ---------- + + it('--plan-from-dir: a single file with 3 distinct problems reports ALL of them in one pass', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-multi-')); + writeFileSync(join(dir, 'multi.json'), MULTI_PROBLEM_PLAN, 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + checked: number; + valid: number; + issues: Array<{ file: string; field: string; reason: string }>; + }; + expect(report.checked).toBe(1); + expect(report.valid).toBe(0); + const fields = report.issues.map(issue => issue.field); + expect(fields).toContain('name'); + expect(fields).toContain('priority'); + expect(fields).toContain('planSteps[0].type'); + expect(report.issues.length).toBeGreaterThanOrEqual(3); + // All three problems are attributed to the ONE file, not scattered. + expect(report.issues.every(issue => issue.file === 'multi.json')).toBe(true); + }); + + it('--plan-from (single file, not a directory) also collects every problem in the file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-single-')); + const file = join(dir, 'plan.json'); + writeFileSync(file, MULTI_PROBLEM_PLAN, 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, planFrom: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + checked: number; + issues: Array<{ field: string }>; + }; + expect(report.checked).toBe(1); + const fields = report.issues.map(issue => issue.field); + expect(fields).toContain('name'); + expect(fields).toContain('priority'); + expect(fields).toContain('planSteps[0].type'); + }); + + it('--steps: a file with 2 distinct problems across different steps reports BOTH', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-steps-')); + const file = join(dir, 'steps.json'); + writeFileSync(file, MULTI_PROBLEM_STEPS, 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, steps: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + checked: number; + issues: Array<{ field: string }>; + }; + expect(report.checked).toBe(1); + const fields = report.issues.map(issue => issue.field); + expect(fields).toContain('planSteps[0].type'); + expect(fields).toContain('planSteps[1].description'); + }); + + it('--steps: 201 steps (over the cap) with a bad first element reports the cap issue AND the per-element issues together', async () => { + // Regression: a length/cap violation must + // NOT short-circuit per-element checking — `stepsRaw` is still a real, + // iterable array even when it's over MAX_PLAN_STEPS (200), so a bad + // step 0 must be reported in the SAME pass as the cap violation, not + // instead of it. + const steps: unknown[] = [{ type: 'hover', description: 123 }]; + for (let i = 1; i < 201; i += 1) { + steps.push({ type: 'action', description: `ok step ${i}` }); + } + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-steps-overcap-')); + const file = join(dir, 'steps.json'); + writeFileSync(file, JSON.stringify({ planSteps: steps }), 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, steps: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + checked: number; + issues: Array<{ field: string; reason: string }>; + }; + expect(report.checked).toBe(1); + const fields = report.issues.map(issue => issue.field); + // The cap violation AND both step-0 problems must all be present. + expect(fields).toContain('planSteps'); + expect(fields).toContain('planSteps[0].type'); + expect(fields).toContain('planSteps[0].description'); + expect(report.issues.length).toBeGreaterThanOrEqual(3); + const capIssue = report.issues.find(issue => issue.field === 'planSteps'); + expect(capIssue?.reason).toContain('at most 200 steps'); + }); + + it('--plan-from: 201 planSteps (over the cap) with a bad first element reports the cap issue AND the per-element issues together', async () => { + // Same regression, exercised through the plan-shape path (collectPlanIssues) + // rather than the steps-shape path (collectPlanStepsIssues) — this one + // never had the bug (it already iterated unconditionally), but the + // coordinator asked to check every collect* function for the same + // pattern, so this locks in that it stays correct. + const planSteps: unknown[] = [{ type: 'hover', description: 123 }]; + for (let i = 1; i < 201; i += 1) { + planSteps.push({ type: 'action', description: `ok step ${i}` }); + } + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-plan-overcap-')); + const file = join(dir, 'plan.json'); + writeFileSync( + file, + JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Over-cap plan', + planSteps, + }), + 'utf8', + ); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, planFrom: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { issues: Array<{ field: string; reason: string }> }; + const fields = report.issues.map(issue => issue.field); + expect(fields).toContain('planSteps'); + expect(fields).toContain('planSteps[0].type'); + expect(fields).toContain('planSteps[0].description'); + const capIssue = report.issues.find(issue => issue.field === 'planSteps'); + expect(capIssue?.reason).toContain('at most 200 steps'); + }); + + it('--plans (JSONL): a single line with multiple problems reports ALL of them, prefixed with specs[N].', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-jsonl-multi-')); + const file = join(dir, 'plans.jsonl'); + writeFileSync(file, `${MULTI_PROBLEM_PLAN}\n`, 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, plans: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + issues: Array<{ field: string; file: string }>; + }; + const fields = report.issues.map(issue => issue.field); + expect(fields).toContain('specs[0].name'); + expect(fields).toContain('specs[0].priority'); + expect(fields).toContain('specs[0].planSteps[0].type'); + expect(report.issues.every(issue => issue.file === `${file}:1`)).toBe(true); + }); }); describe('runTestWaitMany', () => { @@ -3552,6 +3815,329 @@ describe('runTestWaitMany', () => { ).catch((error: unknown) => error); expect(rejection).toMatchObject({ exitCode: 3 }); }); + + // ------------------------------------------------------------------------- + // RATE_LIMITED per-member handling. The 429 is returned as a real HTTP + // response (never thrown directly) so these exercise http.ts's own internal + // RATE_LIMITED retry-then-throw first, which is what makes reaching the + // command-level outer loop meaningful. `retry-after: 0` keeps both budgets + // fast; the header still yields a defined `retryAfterMs` (clamped to ≥1s), + // which is the signal `isTransientRateLimit` reads. + // ------------------------------------------------------------------------- + + const rateLimited = (opts: { retryAfter?: number; message?: string }): Response => + new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: opts.message ?? 'Run trigger rate limit exceeded: 60 per minute per key.', + nextAction: '', + requestId: 'req_rl', + details: {}, + }, + }), + { + status: 429, + headers: { + 'content-type': 'application/json', + ...(opts.retryAfter !== undefined ? { 'retry-after': String(opts.retryAfter) } : {}), + }, + }, + ); + + const jsonOk = (body: unknown): Response => + new Response(JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + + it('retries a throttled poll and still reports the real verdict (exit 0, retry named on stderr)', async () => { + const { credentialsPath } = makeCreds(); + let rateLimitedResponses = 0; + // Throttle every poll until http.ts has exhausted its own budget once, then + // let the run through: without the outer retry the member is a poll error + // and the invocation exits 7 even though the run passed. + const fetchImpl = (async () => { + if (rateLimitedResponses < 4) { + rateLimitedResponses += 1; + return rateLimited({ retryAfter: 0 }); + } + return jsonOk(terminalRun('run_slow', 'passed')); + }) as unknown as typeof globalThis.fetch; + + const out: string[] = []; + const errs: string[] = []; + const payload = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_slow'], + timeoutSeconds: 600, + maxConcurrency: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => errs.push(line), + sleep: () => Promise.resolve(), + }, + ); + expect(payload.summary).toMatchObject({ passed: 1, errors: 0 }); + expect(errs.join('\n')).toContain('[wait] run_slow — rate limited (attempt 1/3)'); + }); + + it('exits 11 (not 7) when a persistent throttle is the ONLY thing that went wrong', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = (async () => + rateLimited({ retryAfter: 0 })) as unknown as typeof globalThis.fetch; + const errs: string[] = []; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_a', 'run_b'], + timeoutSeconds: 600, + maxConcurrency: 2, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => errs.push(line), + sleep: () => Promise.resolve(), + }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 11 }); + expect((rejection as Error).message).toContain('rate limited on 2 of 2 runs'); + // Every member spent its full outer budget before the escalation. + expect(errs.filter(l => l.includes('attempt 3/3')).length).toBe(2); + }); + + it('reports a TIMEOUT (exit 7), not a rate limit, when the shared deadline is reached during a backoff', async () => { + // The escalation claims "nothing else went wrong". An invocation that spent + // its entire `--timeout` inside rate-limit backoff HAS had something else go + // wrong — the wait budget ran out — so exit 11 would be a false claim. + // `sleep` advances a fake clock past the deadline instead of resolving free. + const { credentialsPath } = makeCreds(); + let now = Date.now(); + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => now); + try { + // `retry-after: 0` keeps http.ts's OWN retry chain (real timers) instant. + // Each request advances the fake clock, so the budget is consumed by the + // POLL rather than by the backoff — which is what forces the deadline check + // inside the clamp (`clampedRetryMs <= 0`) rather than the one at the top of + // the loop, i.e. the branch under test. + const fetchImpl = (async () => { + now += 1200; + return rateLimited({ retryAfter: 0 }); + }) as unknown as typeof globalThis.fetch; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_slowpoke'], + timeoutSeconds: 2, + maxConcurrency: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + // Every sleep burns real budget on the fake clock. + sleep: (ms: number) => { + now += ms; + return Promise.resolve(); + }, + }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + expect((rejection as Error).message).toContain('timed out'); + } finally { + nowSpy.mockRestore(); + } + }); + + it('a Ctrl-C during a rate-limit backoff still detaches gracefully (DEV-331), not a hard exit', async () => { + // `pollRunUntilTerminal` disarms the graceful scope in its own `finally`, so + // the outer backoff has to re-arm it — otherwise the sleep is a window where + // the first signal hard-exits with empty stdout. + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const fetchImpl = (async () => + rateLimited({ retryAfter: 0 })) as unknown as typeof globalThis.fetch; + const out: string[] = []; + const errs: string[] = []; + // Fire the signal from inside the OUTER backoff specifically. The stderr line + // is the unambiguous marker that we are in that window (and not in one of the + // poll loop's own sleeps, which share the same injected `sleep`). + let inOuterBackoff = false; + let armedDuringBackoff: boolean | undefined; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_interrupted'], + timeoutSeconds: 600, + maxConcurrency: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => { + errs.push(line); + if (line.includes('rate limited (attempt')) inOuterBackoff = true; + }, + shutdown, + sleep: () => { + if (inOuterBackoff) { + // THE assertion that distinguishes the fix. `installSignalHandlers` + // branches on `isArmed`: armed ⇒ abort and let the wait paths own the + // detach; disarmed ⇒ immediate `process.exit(130)` with empty stdout. + // The in-process abort below would be picked up by the NEXT poll + // either way, so only the armed flag proves the window is covered. + armedDuringBackoff = shutdown.isArmed; + shutdown.interrupt('SIGINT'); + } + return Promise.resolve(); + }, + }, + ).catch((error: unknown) => error); + expect(inOuterBackoff).toBe(true); + expect(armedDuringBackoff).toBe(true); + + expect(rejection).toBeInstanceOf(InterruptError); + // The DEV-331 contract: stdout stays parseable and names the still-running id. + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + }; + expect(payload.results[0]).toMatchObject({ runId: 'run_interrupted', status: 'running' }); + expect(errs.join('\n')).toContain('run_interrupted'); + }); + + it('declines to escalate when the caller repeated a run id (a real failure must not be masked)', async () => { + // `outcomes` is keyed by runId, so a duplicate has ONE shared entry that the + // last lane to finish overwrites. Without the uniqueness guard a later + // RATE_LIMITED replaces an observed `failed` and the invocation exits 11. + const { credentialsPath } = makeCreds(); + let call = 0; + const fetchImpl = (async () => { + call += 1; + // First lane sees a terminal failure; every later poll is throttled. + if (call === 1) return jsonOk(terminalRun('run_dup', 'failed')); + return rateLimited({ retryAfter: 0 }); + }) as unknown as typeof globalThis.fetch; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_dup', 'run_dup'], + timeoutSeconds: 600, + maxConcurrency: 2, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: () => Promise.resolve(), + }, + ).catch((error: unknown) => error); + expect(rejection).not.toMatchObject({ exitCode: 11 }); + }); + + // NOTE: a guard, not a proof — exit 7 is also what the pre-change code returned + // here. It exists so nobody can widen the escalation to fire whenever ANY + // RATE_LIMITED is present without going red. + it('keeps exit 7 when a throttle is mixed with a non-rate-limit poll error (escalation stays narrow)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = (async (input: unknown) => { + const url = String(input); + if (url.includes('run_gone')) { + return new Response( + JSON.stringify({ + error: { + code: 'NOT_FOUND', + message: 'no such run', + nextAction: 'check the id', + requestId: 'req_x', + details: {}, + }, + }), + { status: 404, headers: { 'content-type': 'application/json' } }, + ); + } + return rateLimited({ retryAfter: 0 }); + }) as unknown as typeof globalThis.fetch; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_throttled', 'run_gone'], + timeoutSeconds: 600, + maxConcurrency: 2, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: () => Promise.resolve(), + }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + }); + + // NOTE: partly a guard — the exit code and the absence of retry logging also + // hold on the pre-change code. What it genuinely proves is the re-mapped CODE + // (`error:INSUFFICIENT_CREDITS`), i.e. that this envelope is structurally + // outside both the new retry loop and the new escalation. + it('does not spend the retry budget on a credit-depletion 429, and does not claim it was rate limited', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = (async () => + rateLimited({ + message: 'Insufficient credits: 2 credit(s) required.', + })) as unknown as typeof globalThis.fetch; + const out: string[] = []; + const errs: string[] = []; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_broke'], + timeoutSeconds: 600, + maxConcurrency: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => errs.push(line), + sleep: () => Promise.resolve(), + }, + ).catch((error: unknown) => error); + // `errors.ts` re-maps a credits-flavoured 429 to INSUFFICIENT_CREDITS before + // the poll catch sees it, so it is structurally excluded from both the retry + // loop and the exit-11 escalation — a depleted wallet can't be waited out. + const payload = JSON.parse(out.join('')) as { results: Array<{ status: string }> }; + expect(payload.results[0]!.status).toBe('error:INSUFFICIENT_CREDITS'); + expect(errs.filter(l => l.includes('rate limited (attempt')).length).toBe(0); + // Still folded into 7 (resumable-error bucket). Left alone deliberately: + // escalating a non-retriable code here would be speculative — a poll of + // `GET /runs/{id}` never charges credits, so this envelope is not reachable + // from a real backend on this path; the case exists only to prove the + // re-mapped code neither retries nor masquerades as a throttle. + expect(rejection).toMatchObject({ exitCode: 7 }); + }); }); describe('runResult', () => { @@ -5577,6 +6163,67 @@ describe('runCreate', () => { ).toBe(true); }); + // `test create --type backend --target-url --run` only soft-advises + // (this test's own [C1 Fix 4] case above) while `test run --all --target-url` + // hard-rejects (exit 5, test.run.spec.ts) for the same underlying condition — + // "the URL override has no effect here". The enforcement difference is + // deliberate and kept as-is; only the cosmetic doubled period + wording is + // fixed here. This test locks in the cosmetic half: neither message may + // contain a doubled `..`, regardless of which enforcement path it's on. + it('neither the create soft-advisory nor the run --all hard-reject double their trailing period', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('test("be", async () => {});'); + const fetchImpl = makeFetch(url => + url.includes('/runs') + ? { + body: { + runId: 'run_dev297', + status: 'queued', + enqueuedAt: '2026-07-16T00:00:00.000Z', + codeVersion: 'v-dev297', + targetUrl: '', + }, + } + : { body: { ...SAMPLE_RESPONSE, testId: 'test_dev297', type: 'backend' } }, + ); + const createStderr: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + type: 'backend', + name: 'be test dev297', + codeFile, + targetUrl: 'https://staging.example.com', + run: true, + wait: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => createStderr.push(line), + }, + ); + const createAdvisory = createStderr.find( + l => l.includes('[advisory]') && l.includes('--target-url'), + ); + expect(createAdvisory).toBeDefined(); + expect(createAdvisory).not.toContain('..'); + + const test = createTestCommand(); + disableExits(test); + const rejection = (await test + .parseAsync(['run', '--all', '--project', 'proj_1', '--target-url', 'https://example.com'], { + from: 'user', + }) + .catch((error: unknown) => error)) as ApiError; + expect(rejection.nextAction).toBeDefined(); + expect(rejection.nextAction).not.toContain('..'); + }); + // Fix 4 — B3: duplicate-name advisory it('Fix 4 — emits advisory on stderr when a test with the same name exists, but still proceeds', async () => { const { credentialsPath } = makeCreds(); @@ -7567,6 +8214,246 @@ describe('runCreateFromPlan', () => { expect(errText).toContain('--name'); }); + // --------------------------------------------------------------------------- + // Teach-the-schema validation errors (plan-schema discoverability + // trio: errors / docs+help / schema+template). + // --------------------------------------------------------------------------- + + it('a top-level array gets a dedicated "one test per file" message pointing at create-batch', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile([FE_PLAN, FE_PLAN]); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + nextAction: expect.stringContaining('a plan file holds ONE test as a single JSON object'), + }); + await expect( + runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + nextAction: expect.stringContaining('test create-batch --plans'), + }); + await expect( + runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + nextAction: expect.stringContaining('--plan-from-dir'), + }); + }); + + it('steps nested under `plan.steps` (the Copilot hallucination) hints at top-level `planSteps`', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + projectId: 'project_alice', + type: 'frontend', + name: 'x', + plan: { steps: [{ type: 'action', description: 'go' }] }, + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'planSteps' }), + nextAction: expect.stringContaining('Did you mean `planSteps`?'), + }); + }); + + it('a bare top-level `steps` array also triggers the `planSteps` hint', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + projectId: 'project_alice', + type: 'frontend', + name: 'x', + steps: [{ type: 'action', description: 'go' }], + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + nextAction: expect.stringContaining('Did you mean `planSteps`?'), + }); + }); + + it('a plan with planSteps simply absent (no plan/steps hint fields either) keeps the generic message', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ projectId: 'project_alice', type: 'frontend', name: 'x' }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + details: expect.objectContaining({ field: 'planSteps' }), + nextAction: expect.not.stringContaining('Did you mean'), + }); + }); + + it('a missing projectId AND a supplied --project flag appends the ignored-flag note to the SAME error', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + type: 'frontend', + name: 'x', + planSteps: [{ type: 'action', description: 'go' }], + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const stderrLines: string[] = []; + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + ignoredFlags: ['--project'], + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: l => stderrLines.push(l) }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'projectId' }), + nextAction: expect.stringContaining( + 'note: with --plan-from, --project is ignored; all fields live inside the file.', + ), + }); + // L1778 regression guard: the separate ignored-flags stderr warning still + // must not precede/accompany a validation failure. + expect(stderrLines.join(' ')).not.toContain('warning: --plan-from'); + }); + + it('the ignored-flag note is NOT appended when the flag was not actually supplied', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + type: 'frontend', + name: 'x', + planSteps: [{ type: 'action', description: 'go' }], + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + await expect( + runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + ignoredFlags: [], // --project was NOT supplied on the command line + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + details: expect.objectContaining({ field: 'projectId' }), + nextAction: expect.not.stringContaining('note: with --plan-from'), + }); + }); + + it('a `{{VAR}}` placeholder in a step description is a non-fatal [advisory], not a validation error', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + ...FE_PLAN, + planSteps: [ + { type: 'action', description: 'log in as {{LOGIN_USER}}' }, + { type: 'assertion', description: 'no placeholder here' }, + ], + }); + let posted = false; + const fetchImpl = makeFetch(() => { + posted = true; + return { body: SAMPLE_RESPONSE }; + }); + const stderrLines: string[] = []; + const res = await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: l => stderrLines.push(l) }, + ); + expect(res).toEqual(SAMPLE_RESPONSE); + expect(posted).toBe(true); + const errText = stderrLines.join(' '); + expect(errText).toContain('[advisory]'); + // `.description` must attach to EACH + // flagged path, not just the last one in a joined list. + expect(errText).toContain('planSteps[0].description'); + expect(errText).toContain('contains a'); + expect(errText).toContain('{{...}}'); + expect(errText).not.toContain('planSteps[1]'); + expect(errText).toContain('project update'); + }); + + it('multiple flagged steps (0 and 2) each read `planSteps[N].description`, not a shared trailing suffix', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + ...FE_PLAN, + planSteps: [ + { type: 'action', description: 'log in as {{LOGIN_USER}}' }, + { type: 'action', description: 'no placeholder here' }, + { type: 'assertion', description: 'verify the {{PRODUCT_NAME}} banner' }, + ], + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const stderrLines: string[] = []; + await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: l => stderrLines.push(l) }, + ); + const errText = stderrLines.join(' '); + // Both flagged paths must EACH carry their own `.description` suffix — + // NOT `planSteps[0], planSteps[2].description` (misattributes to only + // the last entry). + expect(errText).toContain('planSteps[0].description'); + expect(errText).toContain('planSteps[2].description'); + expect(errText).not.toMatch(/planSteps\[0\],\s*planSteps\[2\]\.description/); + expect(errText).not.toContain('planSteps[1]'); + // Plural grammar for 2+ flagged steps. + expect(errText).toContain('contain a'); + }); + + it('the placeholder advisory still fires under --dry-run (the agent-iteration loop must not regress)', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile({ + ...FE_PLAN, + planSteps: [{ type: 'action', description: 'log in as {{LOGIN_USER}}' }], + }); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const stderrLines: string[] = []; + await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile, dryRun: true }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: l => stderrLines.push(l) }, + ); + expect(stderrLines.join(' ')).toContain('[advisory]'); + }); + + it('a plan with no placeholders never emits the advisory', async () => { + const { credentialsPath } = makeCreds(); + const planFile = writePlanFile(FE_PLAN); + const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); + const stderrLines: string[] = []; + await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: l => stderrLines.push(l) }, + ); + expect(stderrLines.join(' ')).not.toContain('[advisory]'); + }); + it('rejects a plan with an invalid step type', async () => { const { credentialsPath } = makeCreds(); const planFile = writePlanFile({ @@ -8596,7 +9483,7 @@ describe('Fix 5 — dashboardUrl emission', () => { it('runCreate: JSON mode includes dashboardUrl when API URL is prod', async () => { // Use prod API URL → resolvePortalUrl returns a URL - const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); const codeFile = writeCodeFile('test("dash", async () => {});'); const fetchImpl = makeFetch((_url, init) => { if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; @@ -8623,7 +9510,7 @@ describe('Fix 5 — dashboardUrl emission', () => { }); it('runCreate: text mode emits Dashboard: line to stderr when API URL is prod', async () => { - const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); const codeFile = writeCodeFile('test("dash", async () => {});'); const fetchImpl = makeFetch((_url, init) => { if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; @@ -8653,7 +9540,7 @@ describe('Fix 5 — dashboardUrl emission', () => { }); it('runCreate: no dashboardUrl when API URL is unknown (localhost)', async () => { - const { credentialsPath } = makeCreds('sk-test', 'http://localhost:13502'); + const { credentialsPath } = makeCreds('sk-user-test', 'http://localhost:13502'); const codeFile = writeCodeFile('test("dash", async () => {});'); const fetchImpl = makeFetch((_url, init) => { if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; @@ -8738,7 +9625,7 @@ describe('Fix 5 — dashboardUrl emission', () => { writeFileSync(path, JSON.stringify(plan), 'utf8'); return path; } - const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); const planFile = writePlanFileDash({ projectId: 'proj_dash_plan', type: 'frontend', @@ -8828,7 +9715,7 @@ describe('Fix 5 — dashboardUrl emission', () => { // R3a: dashboardUrl in create --run JSON envelope it('runCreate --run: dashboardUrl is included in the merged { ...create, run } JSON envelope', async () => { // prod API URL so resolvePortalUrl maps correctly - const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); const codeFile = writeCodeFile('test("chain", async () => {});'); const CREATE_RESP = { testId: 'test_chain_01', @@ -8901,7 +9788,7 @@ describe('Fix 5 — dashboardUrl emission', () => { return path; } // Use prod API URL - const { credentialsPath } = makeCreds('sk-test', 'https://api.testsprite.com'); + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); const spec = { projectId: 'proj_batch', type: 'frontend' as const, diff --git a/src/commands/test.ts b/src/commands/test.ts index 6cb692b..b7fe1af 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -23,6 +23,7 @@ import { assertContextIntegrity, buildMeta, pickCodeExtension, + pickVideoExtension, resolveBundleDir, stepFilenamePrefix, writeBundle, @@ -56,6 +57,7 @@ import { import { REQUEST_TIMEOUT_DEFAULT_MS, REQUEST_TIMEOUT_MAX_MS } from '../lib/http.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; +import { VERSION } from '../version.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { fetchSinglePage, @@ -71,6 +73,7 @@ import type { RunStepDto, TriggerRunResponse, RerunResponse, + RerunAdvisory, BatchRerunResponse, BatchRerunAccepted, BatchRerunClosureByProject, @@ -261,13 +264,7 @@ export type CliVerdict = 'passed' | 'failed' | 'blocked'; /** execution LIFECYCLE (where the test is in its run lifecycle). */ export type CliExecutionStatus = - | 'draft' - | 'ready' - | 'queued' - | 'running' - | 'completed' - | 'cancelled' - | 'unknown'; + 'draft' | 'ready' | 'queued' | 'running' | 'completed' | 'cancelled' | 'unknown'; /** §6.5 LatestResult wire shape. All correlation fields are required. */ export interface CliLatestResult { @@ -487,6 +484,32 @@ function interruptDetachMessage(err: InterruptError, runIds: string[]): string { ); } +/** + * The honest-detach stderr line for a RATE_LIMITED (429) hit during a + * `--wait` poll. The backend's pre-auth rate limiter is an in-process LRU + * keyed on `request.ip`, checked before any DB contact — it can trip on a + * shared egress IP (a CI runner, a NAT) and then 429 otherwise-valid keys + * from that IP for its window. The run itself was already triggered (and is + * already billed) and keeps executing server-side regardless of this local + * poll giving up — mirrors `interruptDetachMessage` so the re-attach and + * cancel commands read identically across every detach reason. + */ +function rateLimitedDetachMessage(err: ApiError, runIds: string[]): string { + const retrySuffix = + err.retryAfterMs !== undefined + ? `; the server asked to retry after ~${Math.ceil(err.retryAfterMs / 1000)}s` + : ''; + const subject = + runIds.length === 1 + ? `Run ${runIds[0]} is still executing on the server and will keep running (and billing) until it finishes.` + : `${runIds.length} runs are still executing on the server and will keep running (and billing) until they finish.`; + return ( + `Rate limited by the server (HTTP 429)${retrySuffix}. ${subject}\n` + + ` Re-attach with: testsprite test wait ${runIds.join(' ')}\n` + + ` Cancel with: testsprite test cancel ${runIds.join(' ')}` + ); +} + type CommonOptions = FactoryCommonOptions; interface ListOptions extends CommonOptions { @@ -1252,6 +1275,19 @@ function renderPlanPutText(response: CliPutPlanStepsResponse): string { * 256 KB (vs. 350 KB for code) per the piece-6 spec. */ function readPlanStepsFileGuarded(path: string): CliPlanStep[] { + return assertPlanStepsShape(parsePlanStepsFile(path)); +} + +/** + * File I/O + JSON.parse half of `readPlanStepsFileGuarded`, split out so + * `test lint` can reuse the SAME stat/size/read/parse guards while + * substituting the collect-all `collectPlanStepsIssues` shape check for the + * throw-on-first `assertPlanStepsShape` that `test plan put` still uses. A + * failure here (missing file, oversize, invalid JSON syntax) is always a + * single fatal issue either way — there is nothing left to validate once the + * file itself can't be read or parsed. + */ +function parsePlanStepsFile(path: string): unknown { const absolute = resolveAbsolute(path); let stat; @@ -1292,15 +1328,12 @@ function readPlanStepsFileGuarded(path: string): CliPlanStep[] { throw localValidationError('steps', `cannot read ${path}: ${reason}`); } - let parsed: unknown; try { - parsed = JSON.parse(raw); + return JSON.parse(raw); } catch (err) { const reason = err instanceof Error ? err.message : 'unknown error'; throw localValidationError('steps', `not valid JSON: ${reason}`); } - - return assertPlanStepsShape(parsed); } /** @@ -1335,6 +1368,62 @@ function assertPlanStepsShape(parsed: unknown): CliPlanStep[] { return stepsRaw as CliPlanStep[]; } +/** + * Collect-all counterpart to `assertPlanStepsShape` — same field checks + * (reusing the identical `requireArrayLength`/`requireEnum`/`requireString` + * helpers so wording never drifts), but continues past the first failing + * field instead of throwing. Used exclusively by `test lint`: a + * `--steps` file with several bad steps previously cost one fix-and-rerun + * cycle per step. `assertPlanStepsShape` itself is UNCHANGED and stays + * throw-on-first — `test plan put` only needs the first blocking error per + * network round-trip. + */ +function collectPlanStepsIssues(parsed: unknown): Array<{ field: string; reason: string }> { + const issues: Array<{ field: string; reason: string }> = []; + const check = (validate: () => void): void => { + try { + validate(); + } catch (err) { + issues.push(toLintIssue(err)); + } + }; + + let stepsRaw: unknown; + if (Array.isArray(parsed)) { + stepsRaw = parsed; + } else if (typeof parsed === 'object' && parsed !== null) { + stepsRaw = (parsed as Record).planSteps; + } else { + issues.push({ field: 'steps', reason: 'must be a JSON object with a `planSteps` array' }); + return issues; + } + + check(() => + requireArrayLength('planSteps', stepsRaw, { min: 1, max: MAX_PLAN_STEPS, itemNoun: 'step' }), + ); + // A length/cap violation (e.g. 201 steps, over + // MAX_PLAN_STEPS) does NOT mean there's nothing left to check — `stepsRaw` + // is still a real, iterable array, so per-element problems must be + // collected too (a 201-step file with a bad step 0 must report the cap + // issue AND planSteps[0].type/description in the SAME pass, not one or the + // other). Only bail when `stepsRaw` isn't an array at all — `requireArrayLength`'s + // structural check failed, so there is genuinely nothing to iterate. + if (!Array.isArray(stepsRaw)) return issues; + + for (let i = 0; i < stepsRaw.length; i += 1) { + const step: unknown = stepsRaw[i]; + if (typeof step !== 'object' || step === null || Array.isArray(step)) { + issues.push({ field: `planSteps[${i}]`, reason: 'must be an object' }); + continue; + } + const s = step as Record; + check(() => requireEnum(`planSteps[${i}].type`, s.type, PLAN_STEP_TYPES)); + check(() => requireString(`planSteps[${i}].description`, s.description)); + } + + return issues; +} + /** * §6.X / M3.2 piece-3 `UpdateTestResponse` shape. `updatedFields` is * the array of top-level fields that changed in this call so JSON @@ -1820,6 +1909,95 @@ export interface CliCreateFromPlanResponse extends CliCreateTestResponse { planSteps?: CliPlanStep[]; } +/** + * Public raw-content URL for + * `schemas/plan.schema.json`, shipped in both the npm package (see + * `package.json` `files`) and the repo. Points at the PUBLIC mirror + * (`TestSprite/testsprite-cli`) since that's what ships to npm consumers; + * the private atlas repo syncs to it via `scripts/make-public-snapshot.sh`. + * + * Pinned to the running CLI's own `v` git tag (the same `VERSION` + * constant `--version` / `doctor` / the update-check registry probe already + * read from `src/version.ts`) rather than the mutable `main` branch — a plan + * file authored against one CLI version should resolve the SAME schema + * forever, not whatever `main` happens to contain when the file is opened + * months later (`main` can gain new required fields between versions). + * + * This is deliberately DIFFERENT from `schemas/plan.schema.json`'s own + * internal `$id`, which stays the canonical `main` URL — `$id` is a schema + * IDENTITY (what this document calls itself, used for cross-referencing), + * not a fetch instruction, so it is intentionally version-independent. + * `PLAN_SCHEMA_URL` is the fetch instruction embedded in generated plan + * files and is intentionally version-PINNED. See DOCUMENTATION.md's "Plan + * file format" section for the same distinction spelled out for humans. + * + * Caveat: the pinned tag only resolves once that version is actually + * released — a locally-built/pre-release checkout may see a 404 until then. + */ +export const PLAN_SCHEMA_URL = `https://raw.githubusercontent.com/TestSprite/testsprite-cli/v${VERSION}/schemas/plan.schema.json`; + +/** + * The SINGLE canonical example plan. This + * exact value (via `PLAN_TEMPLATE_TEXT` below) is: + * - printed to stdout by `test create --plan-template` + * - embedded verbatim in `test create --help` (`PLAN_TEMPLATE_HELP_TEXT`) + * - asserted in tests against `schemas/plan.schema.json` and against + * `assertPlanShape` so the three surfaces can't drift + * + * Deliberately minimal (no optional `description`/`priority`) — this is + * the smallest shape `assertPlanShape` accepts, not a fully-annotated + * showcase (that lives in `skills/testsprite-verify.skill.md`). + */ +export const PLAN_TEMPLATE: CliPlanInput = { + projectId: 'prj_abc123', + type: 'frontend', + name: 'Login rejects an empty password', + planSteps: [ + { + type: 'action', + description: 'Navigate to /login and submit the form with an empty password', + }, + { + type: 'assertion', + description: 'Verify an inline error says the password is required', + }, + ], +}; + +/** `CliPlanInput` plus the optional editor-discoverability hint. */ +export interface PlanFileTemplate extends CliPlanInput { + $schema: string; +} + +/** + * `$schema` is an ordinary extra property from `assertPlanShape`'s point of + * view (no `additionalProperties` check exists on the plan-from path) — it + * validates exactly like a bare `CliPlanInput` while giving editors (VS + * Code's JSON language service, and by extension Copilot inline + * completions) something to resolve for live validation as the file is + * edited. + */ +export const PLAN_TEMPLATE_WITH_SCHEMA: PlanFileTemplate = { + $schema: PLAN_SCHEMA_URL, + ...PLAN_TEMPLATE, +}; + +/** + * Rendered once from the object above (never hand-formatted separately) so + * `test create --plan-template`'s stdout and `test create --help`'s example + * are generated from — not merely modeled on — the same source. + */ +export const PLAN_TEMPLATE_TEXT: string = JSON.stringify(PLAN_TEMPLATE_WITH_SCHEMA, null, 2); + +/** `test create --help` after-text. Wraps `PLAN_TEMPLATE_TEXT` unmodified. */ +const PLAN_TEMPLATE_HELP_TEXT = + '\nPlan file format (--plan-from ) — minimal valid example:\n\n' + + `${PLAN_TEMPLATE_TEXT}\n\n` + + 'Print this exact skeleton: testsprite test create --plan-template\n' + + 'Validate offline (no network): testsprite test create --plan-from --dry-run\n' + + 'Multiple tests: test create-batch --plans | --plan-from-dir \n' + + 'Full field reference: DOCUMENTATION.md -> "Plan file format"\n'; + /** Per-spec result from `POST /tests/batch`. */ export interface CliBatchSpecResult { /** Position of the spec in the input JSONL, preserved across the response. */ @@ -1905,6 +2083,37 @@ const MAX_BATCH_RERUN_IDS = 50; * the caller can warn the operator that a shared BE producer/teardown was * triggered more than once. */ +/** + * Dedupe `RerunAdvisory[]` by `feature`+`message`. Used to aggregate + * `advisories` across chunked batch-rerun dispatch requests (initial dispatch + * + D3 deferred-retry attempts) into a single list, and to collapse repeated + * per-attempt advisories in `test flaky` into a single summary line. + */ +function dedupeRerunAdvisories(entries: RerunAdvisory[]): RerunAdvisory[] { + const seen = new Map(); + for (const entry of entries) { + seen.set(`${entry.feature}|${entry.message}`, entry); + } + return [...seen.values()]; +} + +/** + * Print one `[advisory]` stderr line per entry in `advisories`. Mirrors the + * existing style of the other rerun advisories (auto-heal engaged / not + * applied, BE rerun history). No-op when `advisories` is absent or empty — + * this is the common case (every V2 response, every V3 response that did not + * request an autoHeal:false opt-out). + */ +function emitRerunAdvisories( + stderrFn: (line: string) => void, + advisories: RerunAdvisory[] | undefined, +): void { + if (!advisories || advisories.length === 0) return; + for (const advisory of advisories) { + stderrFn(`[advisory] ${advisory.message}`); + } +} + function dedupeBatchRerunAccepted(entries: BatchRerunAccepted[]): { deduped: BatchRerunAccepted[]; droppedCount: number; @@ -1982,6 +2191,69 @@ export const BATCH_RUN_RATE_LIMIT = 50; export const BATCH_RUN_RATE_WINDOW_MS = 60_000; /** Maximum number of outer RATE_LIMITED retries inside the batch fan-out (beyond HTTP-layer retries). */ export const BATCH_RUN_RATE_MAX_OUTER_RETRIES = 5; +/** + * Maximum number of outer RATE_LIMITED retries per member inside the multi-id + * `test wait` fan-out (beyond HTTP-layer retries). Lower than the trigger + * fan-out's budget on purpose: a throttled *poll* costs nothing but latency and + * the run is already executing, so a couple of Retry-After-length backoffs is + * enough to ride out one limiter window — burning the whole `--timeout` on + * backoff instead of reporting the throttle would be worse than reporting it. + */ +export const WAIT_POLL_RATE_MAX_OUTER_RETRIES = 3; + +/** + * Backoff to honour before retrying a `RATE_LIMITED` response, in ms. + * + * Precedence: `ApiError.retryAfterMs` (set by `HttpClient` from the HTTP + * `Retry-After` header, already clamped to [1s, 300s]) → `details.retryAfterSeconds` + * from the envelope body, capped at 120s → 60s. Callers clamp the result to their + * own remaining deadline; this function only decides "how long does the server + * want us to wait." + * + * Shared by the `test run --all` trigger fan-out and the multi-id `test wait` + * poll fan-out so the two can't drift — they were hand-copies of the same + * precedence rule. + */ +/** + * Sleep that a termination signal cuts short by rejecting with the signal's + * `InterruptError`, so the caller's existing DEV-331 catch owns the detach UX. + * + * Mirrors `HttpClient.sleepBeforeRetry` — an in-flight backoff must not be a + * window where a first Ctrl-C hard-exits with empty stdout. The caller is + * responsible for having ARMED the shutdown scope; a disarmed controller never + * aborts its signal, so this would otherwise sleep straight through the signal. + */ +export function sleepUntilOrInterrupt( + ms: number, + signal: AbortSignal | undefined, + sleep: (ms: number) => Promise, +): Promise { + if (signal === undefined) return sleep(ms); + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + sleep(ms).then( + () => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, + err => { + signal.removeEventListener('abort', onAbort); + reject(err instanceof Error ? err : new Error(String(err))); + }, + ); + }); +} + +export function resolveRateLimitRetryMs(err: ApiError): number { + if (err.retryAfterMs !== undefined) return err.retryAfterMs; + const retryAfterSec = err.getDetail( + 'retryAfterSeconds', + (v): v is number => typeof v === 'number' && v > 0, + ); + return Math.min((retryAfterSec ?? 60) * 1000, 120_000); +} /** * D3: max automatic retry attempts for deferred tests under `--wait`. @@ -2074,6 +2346,49 @@ interface CreateFromPlanOptions extends CommonOptions { ignoredFlags?: string[]; } +/** Matches `{{ANYTHING}}`-style template placeholders in a step description. */ +const PLACEHOLDER_PATTERN = /\{\{.*?\}\}/; + +/** + * Indices of `planSteps` whose `description` contains a + * `{{...}}`-style placeholder. Structurally valid (schema-and-validator + * agree these plans pass) — this is a content-quality signal, not a shape + * violation, so it's surfaced separately as a non-fatal advisory rather + * than folded into `assertPlanShape`. + */ +function findPlaceholderStepIndices(plan: CliPlanInput): number[] { + const indices: number[] = []; + plan.planSteps.forEach((step, i) => { + if (PLACEHOLDER_PATTERN.test(step.description)) indices.push(i); + }); + return indices; +} + +/** + * Non-fatal `[advisory]` when one or more `planSteps[].description` values + * contain a `{{...}}`-style placeholder — the most common false assumption + * agent-authored plans make (that the CLI does variable substitution). The + * CLI does none: the browser agent types the literal braces into the + * field. Points at storing credentials on the project instead, which is + * the actual mechanism for injecting auth into a run. + */ +function emitPlaceholderAdvisory(plan: CliPlanInput, stderrFn: (line: string) => void): void { + const indices = findPlaceholderStepIndices(plan); + if (indices.length === 0) return; + // Each path must read + // `planSteps[N].description` on its OWN — appending `.description` once + // after a joined `planSteps[0], planSteps[2]` list misattributed it to + // only the last entry. + const paths = indices.map(i => `planSteps[${i}].description`).join(', '); + const verb = indices.length === 1 ? 'contains' : 'contain'; + stderrFn( + `[advisory] ${paths} ${verb} a ` + + '`{{...}}`-style placeholder; the CLI does no variable substitution — ' + + 'the browser agent will type the literal braces. Store login credentials on the project instead: ' + + '`testsprite project update --username --password `, or Portal -> Project Settings.', + ); +} + /** * `test create --plan-from ` — M3.2 piece-5. * @@ -2108,15 +2423,23 @@ export async function runCreateFromPlan( assertNotLocal(opts.targetUrl); } - const plan = readPlanFromGuarded(opts.planFrom); + const plan = readPlanFromGuarded(opts.planFrom, { ignoredFlags: opts.ignoredFlags }); + + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + // Non-fatal advisory for `{{...}}`-style placeholders in step + // descriptions. The CLI does no variable substitution — the browser agent + // types the literal braces — so this is a content-quality nudge, not a + // validation failure. Fires regardless of --dry-run (dry-run still runs + // full local validation; this advisory is part of that same offline pass). + emitPlaceholderAdvisory(plan, stderrFn); // The plan validated (projectId/type/name/planSteps present). Only NOW // warn that overlapping `test create` flags were ignored — emitting this // before validation made a missing-projectId failure look like the // ignored --project flag was the cause (dogfood L1778). if (opts.ignoredFlags && opts.ignoredFlags.length > 0) { - const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - stderr( + stderrFn( `warning: --plan-from supplies the test definition; ignoring ${opts.ignoredFlags.join(', ')}. ` + `Edit the plan JSON to change these fields.`, ); @@ -2140,8 +2463,7 @@ export async function runCreateFromPlan( const idempotencyKey = opts.idempotencyKey ?? `cli-create-plan-${randomUUID()}`; if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { - const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); - stderr(`idempotency-key: ${idempotencyKey}`); + stderrFn(`idempotency-key: ${idempotencyKey}`); } const body = { @@ -2160,7 +2482,6 @@ export async function runCreateFromPlan( // The plan's projectId + name are available after validation above. Skip // under dry-run (no network calls); swallow all errors (advisory only). if (!opts.dryRun) { - const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); await emitDupNameAdvisoryIfNeeded(client, plan.projectId, plan.name, stderrFn); } @@ -2209,7 +2530,6 @@ export async function runCreateFromPlan( } else { out.print(response, data => renderCreateText(data as CliCreateTestResponse)); if (planDashboardUrl !== undefined) { - const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); stderrFn(`Dashboard: ${planDashboardUrl}`); } } @@ -2225,7 +2545,27 @@ export async function runCreateFromPlan( * obvious oversize files BEFORE loading them into V8's heap. For * plans the cap is 256 KB (vs. 350 KB for code). */ -function readPlanFromGuarded(path: string): CliPlanInput { +function readPlanFromGuarded( + path: string, + context: { ignoredFlags?: string[] } = {}, +): CliPlanInput { + return assertPlanShape(parsePlanFile(path), context); +} + +/** + * File I/O + JSON.parse half of `readPlanFromGuarded`, split out so `test + * lint` can reuse the SAME stat/size/read/parse guards while + * substituting the collect-all `collectPlanIssues` shape check for the + * throw-on-first `assertPlanShape` that `create`/`create-batch` still use. A + * failure here (missing file, oversize, invalid JSON syntax) is always a + * single fatal issue either way — there is nothing left to validate once the + * file itself can't be read or parsed. + * + * Stat-first guard mirrors piece-2's `readCodeFileGuarded` — reject + * obvious oversize files BEFORE loading them into V8's heap. For + * plans the cap is 256 KB (vs. 350 KB for code). + */ +function parsePlanFile(path: string): unknown { const absolute = resolveAbsolute(path); let stat; @@ -2266,15 +2606,49 @@ function readPlanFromGuarded(path: string): CliPlanInput { throw localValidationError('plan-from', `cannot read ${path}: ${reason}`); } - let parsed: unknown; try { - parsed = JSON.parse(raw); + return JSON.parse(raw); } catch (err) { const reason = err instanceof Error ? err.message : 'unknown error'; throw localValidationError('plan-from', `not valid JSON: ${reason}`); } +} - return assertPlanShape(parsed); +/** + * Rethrow a validation error enriched with a note that the + * caller's `--project`/`--type`/`--name` flag was ignored, but ONLY when + * that flag was actually supplied. Keeps the L1778 ordering intact + * (validation still runs, and still throws, before the general + * ignored-flags warning in `runCreateFromPlan`) — this only changes the + * WORDING of the validation error itself so the one hint that would + * explain a missing-field failure lands inside the error the caller + * already sees, instead of depending on a separate warning that (per + * L1778) deliberately fires after validation succeeds. + */ +function appendIgnoredFlagNote(err: ApiError, flag: string): ApiError { + return new ApiError({ + code: err.code, + message: err.message, + nextAction: `${err.nextAction} note: with --plan-from, ${flag} is ignored; all fields live inside the file.`, + requestId: err.requestId, + details: err.details, + }); +} + +/** Runs `fn`, enriching any thrown `ApiError` via {@link appendIgnoredFlagNote} when `flag` was ignored. */ +function requireFieldNotIgnored( + fn: () => T, + flag: string, + ignoredFlags: string[] | undefined, +): T { + try { + return fn(); + } catch (err) { + if (err instanceof ApiError && ignoredFlags?.includes(flag)) { + throw appendIgnoredFlagNote(err, flag); + } + throw err; + } } /** @@ -2283,22 +2657,63 @@ function readPlanFromGuarded(path: string): CliPlanInput { * `create-batch --plans`. Throws `VALIDATION_ERROR` with a typed * `details.field` pointer so callers can fix specific issues without * re-reading the whole file. + * + * `context.ignoredFlags` is populated only on the single + * `--plan-from` path (`test create --plan-from` also received overlapping + * `--project`/`--type`/`--name` flags); batch/dir/JSONL callers never pass + * it, so `requireFieldNotIgnored` below is a no-op for them. + * + * Throw-on-first is intentional here: `create` / `create-batch` POST over + * the network, so surfacing the first blocking error per round-trip is the + * right cost/detail tradeoff. `test lint`'s collect-all sibling is + * `collectPlanIssues` below — do not merge the two; the doc comment on + * `collectPlanIssues` explains why they must stay separate. */ -function assertPlanShape(parsed: unknown, context: { specIndex?: number } = {}): CliPlanInput { +function assertPlanShape( + parsed: unknown, + context: { specIndex?: number; ignoredFlags?: string[] } = {}, +): CliPlanInput { const prefix = context.specIndex !== undefined ? `specs[${context.specIndex}].` : ''; + // A top-level JSON array is the single most common + // agent-authored mistake: a plan file holds exactly ONE test. Give it a + // dedicated message pointing at the batch surfaces, instead of the + // generic "must be a JSON object" (which doesn't say what to do about an + // array). + if (Array.isArray(parsed)) { + throw localValidationError( + `${prefix}plan`, + `a plan file holds ONE test as a single JSON object (got an array of ${parsed.length}). ` + + 'To create many tests use `test create-batch --plans ` or `--plan-from-dir `', + undefined, + 'field', + ); + } + // Every field below is a JSON body path inside the plan file (or // JSONL spec), not a CLI flag — pass `'field'` so the error message // says `Field \`projectId\` is invalid: ...` instead of inventing a // `--projectId` flag the user can't pass. - if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + if (typeof parsed !== 'object' || parsed === null) { throw localValidationError(`${prefix}plan`, 'must be a JSON object', undefined, 'field'); } const obj = parsed as Record; - requireString(`${prefix}projectId`, obj.projectId); - requireEnum(`${prefix}type`, obj.type, ['frontend', 'backend'] as const); - requireString(`${prefix}name`, obj.name); + requireFieldNotIgnored( + () => requireString(`${prefix}projectId`, obj.projectId), + '--project', + context.ignoredFlags, + ); + requireFieldNotIgnored( + () => requireEnum(`${prefix}type`, obj.type, ['frontend', 'backend'] as const), + '--type', + context.ignoredFlags, + ); + requireFieldNotIgnored( + () => requireString(`${prefix}name`, obj.name), + '--name', + context.ignoredFlags, + ); if (obj.description !== undefined && typeof obj.description !== 'string') { throw localValidationError( `${prefix}description`, @@ -2310,6 +2725,21 @@ function assertPlanShape(parsed: unknown, context: { specIndex?: number } = {}): if (obj.priority !== undefined) { requireEnum(`${prefix}priority`, obj.priority, CLI_CREATE_PRIORITIES); } + + // `planSteps` missing is the single most common agent + // hallucination: LLMs (Copilot included) reliably nest steps under + // `plan.steps` or a bare top-level `steps`. Point directly at the fix + // instead of falling through to the generic "is required and must be an + // array" message, which doesn't say WHERE the steps actually belong. + if (obj.planSteps === undefined && (obj.plan !== undefined || obj.steps !== undefined)) { + throw localValidationError( + `${prefix}planSteps`, + 'is required and must be an array. Did you mean `planSteps`? Steps live at the top level: ' + + '`"planSteps": [{ "type": "action" | "assertion", "description": "..." }]`', + undefined, + 'field', + ); + } requireArrayLength(`${prefix}planSteps`, obj.planSteps, { min: 1, max: MAX_PLAN_STEPS, @@ -2333,6 +2763,72 @@ function assertPlanShape(parsed: unknown, context: { specIndex?: number } = {}): return obj as unknown as CliPlanInput; } +/** + * Collect-all counterpart to `assertPlanShape` — same field checks (reusing + * the identical `requireString`/`requireEnum`/`requireArrayLength` helpers + * so wording never drifts), but continues past the first failing field + * instead of throwing. Used exclusively by `test lint` (issue #98 + * follow-up): the throw-on-first + * `assertPlanShape` meant a plan with 6 independent problems reported one at + * a time across 6 fix-and-rerun cycles. `assertPlanShape` itself is + * UNCHANGED — `create`/`create-batch` only need the first blocking error per + * network round-trip, and duplicating the field checks here (rather than + * threading a "collect" flag through the throw-on-first assert) keeps both + * functions simple and matches the existing sibling-validator convention + * already used between `assertPlanShape` and `assertPlanStepsShape`. + */ +function collectPlanIssues( + parsed: unknown, + context: { specIndex?: number } = {}, +): Array<{ field: string; reason: string }> { + const prefix = context.specIndex !== undefined ? `specs[${context.specIndex}].` : ''; + const issues: Array<{ field: string; reason: string }> = []; + const check = (validate: () => void): void => { + try { + validate(); + } catch (err) { + issues.push(toLintIssue(err)); + } + }; + + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + issues.push({ field: `${prefix}plan`, reason: 'must be a JSON object' }); + return issues; + } + const obj = parsed as Record; + + check(() => requireString(`${prefix}projectId`, obj.projectId)); + check(() => requireEnum(`${prefix}type`, obj.type, ['frontend', 'backend'] as const)); + check(() => requireString(`${prefix}name`, obj.name)); + if (obj.description !== undefined && typeof obj.description !== 'string') { + issues.push({ field: `${prefix}description`, reason: 'must be a string when present' }); + } + if (obj.priority !== undefined) { + check(() => requireEnum(`${prefix}priority`, obj.priority, CLI_CREATE_PRIORITIES)); + } + check(() => + requireArrayLength(`${prefix}planSteps`, obj.planSteps, { + min: 1, + max: MAX_PLAN_STEPS, + itemNoun: 'step', + }), + ); + if (Array.isArray(obj.planSteps)) { + for (let i = 0; i < obj.planSteps.length; i += 1) { + const step: unknown = obj.planSteps[i]; + if (typeof step !== 'object' || step === null || Array.isArray(step)) { + issues.push({ field: `${prefix}planSteps[${i}]`, reason: 'must be an object' }); + continue; + } + const s = step as Record; + check(() => requireEnum(`${prefix}planSteps[${i}].type`, s.type, PLAN_STEP_TYPES)); + check(() => requireString(`${prefix}planSteps[${i}].description`, s.description)); + } + } + + return issues; +} + interface CreateBatchOptions extends CommonOptions { /** Path to the JSONL file containing one `CliPlanInput` per line. */ plans: string; @@ -2808,16 +3304,9 @@ async function runBatchRun( // MAJOR 3: use retryAfterMs from the thrown ApiError when available // (set by HttpClient from the HTTP Retry-After header, clamped to // [1s, 300s]). Fall back to details.retryAfterSeconds, then 60 s. - let retryAfterMs: number; - if (err.retryAfterMs !== undefined) { - retryAfterMs = err.retryAfterMs; - } else { - const retryAfterSec = err.getDetail( - 'retryAfterSeconds', - (v): v is number => typeof v === 'number' && v > 0, - ); - retryAfterMs = Math.min((retryAfterSec ?? 60) * 1000, 120_000); - } + // Extracted to `resolveRateLimitRetryMs` so the multi-id `test wait` + // poll fan-out applies the identical precedence. + const retryAfterMs = resolveRateLimitRetryMs(err); // MAJOR 2: clamp to remaining deadline so we don't overshoot the // --wait budget. @@ -3021,7 +3510,9 @@ async function runBatchRun( testId: finalRun.testId, runId: finalRun.runId, status: finalRun.status, - codeVersion: finalRun.codeVersion, + // Runs without a stored code body report `codeVersion: null`; the batch + // envelope uses '' for "unknown", as the trigger-error paths above do. + codeVersion: finalRun.codeVersion ?? '', videoUrl: finalRun.videoUrl, failureKind: finalRun.failureKind, }; @@ -3917,13 +4408,37 @@ export interface CliRunDiff { changedSteps: CliDiffStep[]; } +/** + * The `test diff` exit-code contract, applied identically to a real + * two-run comparison and the `--dry-run` canned sample: the + * result is always printed first (`out.print` already ran by the time this + * is called), then a `verdictChanged` diff throws so the process exits 1. + * Before this helper existed, `--dry-run`'s early `return sample` bypassed + * the check entirely — the canned sample has `verdictChanged: true`, so + * `test diff --dry-run` always exited 0 even though the documented contract + * ("Exit 0 when verdicts match, 1 when they differ") makes no dry-run + * exception. That made the command useless for its stated CI-gate + * pre-verification purpose. + */ +function enforceDiffExitContract(diff: CliRunDiff): void { + if (diff.verdictChanged) { + throw new CLIError( + `verdicts differ: ${diff.runA.runId}=${diff.runA.status} vs ${diff.runB.runId}=${diff.runB.status}`, + 1, + ); + } +} + /** * `test diff ` (issue #124): isolate what regressed between two * runs, the first question when CI goes red ("what changed since the last * green run?"). Pure client-side composition of the existing per-run read * (`GET /runs/{id}?includeSteps=true`); the endpoint accepts any two run-ids, * so a cross-test pair is a WARNING, not an error. Exit 0 when the verdicts - * match, exit 1 when they differ, so the command is CI-scriptable. + * match, exit 1 when they differ, so the command is CI-scriptable — + * `--dry-run` honors the same contract: the canned sample has a + * changed verdict, so `test diff --dry-run` (with no overrides) exits 1, + * same as a real regressed pair would. */ export async function runDiff(opts: DiffOptions, deps: TestDeps = {}): Promise { const out = makeOutput(opts.output, deps); @@ -3956,6 +4471,7 @@ export async function runDiff(opts: DiffOptions, deps: TestDeps = {}): Promise renderRunDiffText(sample)); + enforceDiffExitContract(sample); return sample; } @@ -4024,13 +4540,8 @@ export async function runDiff(opts: DiffOptions, deps: TestDeps = {}): Promise renderRunDiffText(diff)); - if (diff.verdictChanged) { - // Result already printed; the typed exit makes `test diff` a CI gate. - throw new CLIError( - `verdicts differ: ${runA.runId}=${runA.status} vs ${runB.runId}=${runB.status}`, - 1, - ); - } + // Result already printed; the typed exit makes `test diff` a CI gate. + enforceDiffExitContract(diff); return diff; } @@ -4085,6 +4596,26 @@ export interface CliLintReport { issues: CliLintIssue[]; } +/** + * Turn a thrown validation error into a lint issue's `field`+`reason` pair. + * Shared by `runLint`'s file-level failures (bad path, oversize, invalid JSON + * syntax — always a single issue, there's nothing left to validate) and the + * collect-all `collectPlanIssues` / `collectPlanStepsIssues` helpers above + * (one call per field, so every problem in a file is captured, not just the + * first). Preserves the typed envelope's `details.field` / `details.reason` + * verbatim (e.g. `planSteps[2].type`) so a caller sees the exact same pointer + * whether the error surfaced via lint or via `create`. + */ +function toLintIssue(err: unknown): { field: string; reason: string } { + if (err instanceof ApiError) { + return { + field: String(err.getDetail('field') ?? '(file)'), + reason: String(err.getDetail('reason') ?? err.nextAction ?? err.message), + }; + } + return { field: '(file)', reason: err instanceof Error ? err.message : String(err) }; +} + /** * `test lint` (issue #98): validate plan/steps files fully OFFLINE with the * SAME validators the create paths run, but collecting EVERY problem instead @@ -4093,6 +4624,15 @@ export interface CliLintReport { * so authoring a 12-plan directory meant one error per paid round-trip. Zero * network, zero credentials: exit 0 when everything is valid, 5 otherwise, so * it drops into a pre-commit hook or CI step before `create-batch`. + * + * The collection granularity used to be per-FILE, not per-PROBLEM — a single + * plan with 6 independent field errors reported one at a time across 6 + * fix-and-rerun cycles, because each file was validated through the + * throw-on-first `assertPlanShape`/`assertPlanStepsShape`. Every branch below + * now separates "parse the file" (still a single fatal issue on I/O/JSON + * failure — there's nothing left to validate) from "check the parsed shape" + * (routed through `collectPlanIssues` / `collectPlanStepsIssues`, which + * report every failing field in one pass). */ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { const out = makeOutput(opts.output, deps); @@ -4108,36 +4648,39 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise void): void => { + + const lintPlanFile = (file: string, path: string, specIndex?: number): void => { checked += 1; + let parsed: unknown; try { - validate(); + parsed = parsePlanFile(path); } catch (err) { - if (err instanceof ApiError) { - issues.push({ - file, - field: String(err.getDetail('field') ?? '(file)'), - reason: String(err.getDetail('reason') ?? err.nextAction ?? err.message), - }); - } else { - issues.push({ - file, - field: '(file)', - reason: err instanceof Error ? err.message : String(err), - }); - } + issues.push({ file, ...toLintIssue(err) }); + return; + } + for (const issue of collectPlanIssues(parsed, { specIndex })) { + issues.push({ file, ...issue }); + } + }; + + const lintStepsFile = (file: string, path: string): void => { + checked += 1; + let parsed: unknown; + try { + parsed = parsePlanStepsFile(path); + } catch (err) { + issues.push({ file, ...toLintIssue(err) }); + return; + } + for (const issue of collectPlanStepsIssues(parsed)) { + issues.push({ file, ...issue }); } }; if (opts.planFrom !== undefined) { - const planFrom = opts.planFrom; - collect(planFrom, () => void readPlanFromGuarded(planFrom)); + lintPlanFile(opts.planFrom, opts.planFrom); } else if (opts.steps !== undefined) { - const steps = opts.steps; - collect(steps, () => void readPlanStepsFileGuarded(steps)); + lintStepsFile(opts.steps, opts.steps); } else if (opts.planFromDir !== undefined) { const dir = resolveAbsolute(opts.planFromDir); let entries: string[]; @@ -4152,11 +4695,12 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise void readPlanFromGuarded(join(dir, entry))); + lintPlanFile(entry, join(dir, entry)); } } else if (opts.plans !== undefined) { - // JSONL: validate PER LINE so every bad line reports (the create path's - // reader stays throw-on-first; this is the collecting counterpart). + // JSONL: validate PER LINE, and every problem WITHIN each line (the + // create path's reader stays throw-on-first; this is the collecting + // counterpart, both per-line and per-field). const absolute = resolveAbsolute(opts.plans); let content: string; try { @@ -4173,20 +4717,24 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise entry.line.length > 0); if (numberedLines.length === 0) throw localValidationError('plans', 'contains no plan lines'); for (const { line, lineNo } of numberedLines) { - collect(`${opts.plans}:${lineNo}`, () => { - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch { - throw localValidationError( - 'plans', - `line ${lineNo} is not valid JSON`, - undefined, - 'field', - ); - } - assertPlanShape(parsed, { specIndex: lineNo - 1 }); - }); + const file = `${opts.plans}:${lineNo}`; + checked += 1; + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + const err = localValidationError( + 'plans', + `line ${lineNo} is not valid JSON`, + undefined, + 'field', + ); + issues.push({ file, ...toLintIssue(err) }); + continue; + } + for (const issue of collectPlanIssues(parsed, { specIndex: lineNo - 1 })) { + issues.push({ file, ...issue }); + } } } @@ -4312,6 +4860,26 @@ export async function runScaffold( return payload; } +/** + * `test create --plan-template` — prints the canonical minimal + * valid plan file (`PLAN_TEMPLATE_WITH_SCHEMA` / `PLAN_TEMPLATE_TEXT`, + * defined above near `CliPlanInput`) to stdout and exits — before any of + * `test create`'s other flag handling runs. Pure-local: no network, no + * credentials, no filesystem I/O. + * + * Deliberately simpler than `test scaffold --type frontend` (which + * substitutes a live `TESTSPRITE_PROJECT_ID` when set, and supports + * `--out`): `--plan-template`'s job is a single deterministic ground-truth + * document that is BYTE-IDENTICAL every invocation, because it doubles as + * the literal example embedded in `test create --help` and the + * fixture asserted against `schemas/plan.schema.json` in tests. + */ +export function runPlanTemplate(opts: CommonOptions, deps: TestDeps = {}): PlanFileTemplate { + const out = makeOutput(opts.output, deps); + out.print(PLAN_TEMPLATE_WITH_SCHEMA, () => PLAN_TEMPLATE_TEXT); + return PLAN_TEMPLATE_WITH_SCHEMA; +} + export interface OpenOptions extends CommonOptions { testId: string; /** Print the URL only; never spawn a browser (SSH/headless/CI/agents). */ @@ -4578,6 +5146,11 @@ interface ResultHistoryOptions extends CommonOptions { pageSize?: number; /** Opaque cursor from a prior page's `nextCursor`. */ cursor?: string; + /** + * Client-side rerun filter. `true` → only reruns (isRerun), `false` → only + * fresh runs, `undefined` → no filter. Applied to each page after the fetch. + */ + rerun?: boolean; columns?: string; noHeader?: boolean; } @@ -4623,9 +5196,16 @@ export async function runResultHistory( since: sinceIso, }); + // Client-side rerun filter (--rerun / --no-rerun). isRerun is on every row; + // the backend has no rerun filter. Undefined → no filter. Like --source, a + // filtered page can be short/empty while more history exists — the empty-page + // and short-page hints below cover that. + const runs = + opts.rerun === undefined ? resp.runs : resp.runs.filter(r => r.isRerun === opts.rerun); + if (opts.output !== 'text') { - out.print({ runs: resp.runs, nextCursor: resp.nextCursor }, data => JSON.stringify(data)); - return resp; + out.print({ runs, nextCursor: resp.nextCursor }, data => JSON.stringify(data)); + return { ...resp, runs }; } // Text mode rendering @@ -4636,7 +5216,7 @@ export async function runResultHistory( // backend filters AFTER limiting rows (limit-before-filter). Matching runs // may exist on later pages — surface the cursor instead of reporting // "no history". - if (resp.runs.length === 0) { + if (runs.length === 0) { if (resp.nextCursor !== null) { // Filtered-empty page, but more pages exist: prompt user to paginate. const msg = @@ -4660,7 +5240,7 @@ export async function runResultHistory( } const lines: string[] = []; - lines.push(renderRunHistoryTable(resp.runs, { columns: opts.columns, noHeader: opts.noHeader })); + lines.push(renderRunHistoryTable(runs, { columns: opts.columns, noHeader: opts.noHeader })); // Footer: pointer to per-run detail commands. lines.push(''); @@ -4677,14 +5257,14 @@ export async function runResultHistory( // Short-filtered-page hint: non-null nextCursor even though this page was // shorter than requested — "none in THIS window" does not mean end-of-history. - if (resp.nextCursor !== null && resp.runs.length < pageSize) { + if (resp.nextCursor !== null && runs.length < pageSize) { stderr( `[hint] Fewer than ${pageSize} rows returned but more may exist — ` + - `source filter skipped some entries. Pass --cursor ${resp.nextCursor} to continue.`, + `a source/rerun filter skipped some entries. Pass --cursor ${resp.nextCursor} to continue.`, ); } - return resp; + return { ...resp, runs }; } const RUN_HISTORY_TABLE_COLUMNS: ReadonlyArray> = [ @@ -5115,8 +5695,11 @@ function backendResultToRunResponse(result: CliLatestResult, run: RunResponse): retryAfterSeconds: undefined, startedAt: result.startedAt ?? run.startedAt, finishedAt: result.finishedAt ?? run.finishedAt, - codeVersion: run.codeVersion || (result.codeVersion ?? ''), - targetUrl: run.targetUrl || (result.targetUrl ?? ''), + // Neither side having a value stays null (not ''), so the text renderer + // omits the line instead of printing a label with nothing after it. A + // backend run legitimately has no target URL at all. + codeVersion: run.codeVersion || result.codeVersion || null, + targetUrl: run.targetUrl || result.targetUrl || null, // createdFrom / projectId / userId / runId / testId / source / createdAt // inherited from the polled run row via the spread above. failedStepIndex: result.failedStepIndex, @@ -5251,13 +5834,34 @@ function printRunOrChain( } /** - * Enrich a terminal RunResponse with a client-synthesized Portal deep link. - * Emitted only when the wire row carries both projectId and testId (the BE - * testId-fallback path synthesizes rows with an empty projectId — those stay - * unenriched) and the API endpoint maps to a known portal host - * (`resolvePortalUrl` returns undefined otherwise). + * Attach the Portal deep link to a terminal RunResponse. + * + * **A server-provided `dashboardUrl` always wins.** The backend knows two + * things this process cannot: + * + * - **Which store answered.** For a run served from V3 Postgres the + * `/dashboard/tests/…` route we would template reads DynamoDB only, and a + * CLI-created project under the V3-native write path has no DynamoDB row at + * all — so our link is not merely org-less, it cannot render. The server + * sends the V3 test-case page instead, scoped to the workspace that owns the + * run. + * - **The portal origin for this environment.** `resolvePortalUrl` maps only + * the prod host, so against any other endpoint it returns undefined and we + * print nothing. The server reads its own `PORTAL_URL`. + * + * The client computation stays as the fallback so an older backend (no field) + * behaves exactly as before, and so does a V2-served run whose server link the + * backend chose not to emit. */ function withRunDashboardUrl(run: RunResponse, apiUrl: string): RunResponse { + // A server value of `null` is meaningful, not missing: it says "there is no + // correct link for this run" (e.g. a workspace-scoped page the environment's + // portal build does not serve yet). Falling back to our own guess there would + // reintroduce exactly the wrong link the server declined to send — so only an + // ABSENT field (older backend) reopens the client path. + if ('dashboardUrl' in run) { + return run.dashboardUrl ? run : { ...run, dashboardUrl: undefined }; + } if (!run.projectId || !run.testId) return run; const dashboardUrl = resolvePortalUrl(apiUrl, run.projectId, run.testId); return dashboardUrl !== undefined ? { ...run, dashboardUrl } : run; @@ -5488,7 +6092,7 @@ export async function runTestRun( code: 'CONFLICT', message: `Conflict: another run for this test is in flight against a different ` + - `target URL (${inFlightRun.targetUrl}). Your --target-url ${opts.targetUrl} ` + + `target URL (${inFlightRun.targetUrl ?? 'not reported'}). Your --target-url ${opts.targetUrl} ` + `cannot attach to that run. Wait for it to finish ` + `(\`testsprite test wait ${currentRunId}\`) or retry your trigger when ` + `the test is free.`, @@ -5512,8 +6116,9 @@ export async function runTestRun( runId: currentRunId, status: 'queued', enqueuedAt: new Date().toISOString(), - codeVersion: inFlightRun.codeVersion, - targetUrl: inFlightRun.targetUrl, + codeVersion: inFlightRun.codeVersion ?? '', + // The check above proved the in-flight run targets exactly this URL. + targetUrl: opts.targetUrl, }; } else { // D: No --target-url supplied — fetch the in-flight run so the @@ -5685,6 +6290,37 @@ export async function runTestRun( ); throw err; } + // RATE_LIMITED during polling — the backend's pre-auth rate limiter can + // trip on a shared egress IP (CI runner, NAT) and 429 an otherwise-valid + // key for its window; the HTTP layer already retried internally and gave + // up. The run was already triggered (and billed) and keeps executing + // server-side, so this must not be a silent, runId-less death: same + // partial-envelope contract as the RequestTimeoutError branch above, and + // the SAME thrown ApiError is rethrown unchanged so its native exit code + // (11) is preserved — never reclassified to 7. + if (err instanceof ApiError && err.code === 'RATE_LIMITED') { + ticker.finalize(`Run ${triggerResponse.runId} — rate limited by the server`); + const partial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, partial, opts.createContext, data => { + const p = data as typeof partial; + const lines = [ + `runId ${p.runId}`, + `status ${p.status} (rate limited by the server)`, + ]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); + return lines.join('\n'); + }); + stderrFn(rateLimitedDetachMessage(err, [triggerResponse.runId])); + throw err; + } // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1): same partial- // envelope shape as the timeout paths so stdout stays parseable, plus the // honest "keeps running and billing" stderr line. Rethrow → index.ts @@ -5815,6 +6451,7 @@ export async function runTestWaitMany( deps, ); const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + const sleepFn = deps.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); // One shared deadline across every member (the whole point of the shared // pool: `--timeout 600` means the invocation ends within ~600s, not @@ -5827,39 +6464,91 @@ export async function runTestWaitMany( | { kind: 'error'; code: string; exitCode: number }; const pollOne = async (runId: string): Promise => { - // A member dequeued AFTER the shared deadline has passed must not be - // granted a fresh minimum poll window (with --max-concurrency 1 that - // would extend the invocation by ~1s per queued run past --timeout). - const remainingSeconds = Math.ceil((deadlineMs - Date.now()) / 1000); - if (remainingSeconds <= 0) return { kind: 'timeout' }; const resolveAlternate = makeBackendWaitFallback({ client, resolveTestId: run => run.testId, resolveNotBefore: run => run.createdAt, onResolved: () => undefined, }); - try { - const run = await pollRunUntilTerminal(client, runId, { - timeoutSeconds: remainingSeconds, - sleep: deps.sleep, - shutdown: shutdownOf(deps), - onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, - onTick: (run, elapsedMs) => { - const elapsed = Math.round(elapsedMs / 1000); - ticker.update(`Run ${run.runId} — ${run.status} (elapsed=${elapsed}s)`); - }, - resolveAlternate, - }); - return { kind: 'result', run }; - } catch (err) { - if (err instanceof TimeoutError) return { kind: 'timeout' }; - if (err instanceof RequestTimeoutError) throw err; - // Interrupt must reject the fan-out (handled at the collect point), not - // be flattened into a per-member 'error' outcome that would swallow the - // 128+signum exit (DEV-331). - if (err instanceof InterruptError) throw err; - if (err instanceof ApiError) return { kind: 'error', code: err.code, exitCode: err.exitCode }; - return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; + // Outer RATE_LIMITED retry, per member. `http.ts` already retries a 429 + // internally (`MAX_ATTEMPTS_RATE_LIMITED`), so reaching this catch means the + // transport gave up — but a shared-egress 429 (CI runner / NAT tripping the + // backend's ip-keyed pre-auth limiter) is a property of the *window*, not of + // this run, and the run is still executing. Without this loop a single 429 + // ended the whole member's wait, which is what made a healthy run report as + // a poll error. Mirrors the `test run --all` trigger fan-out's outer loop + // (`BATCH_RUN_RATE_MAX_OUTER_RETRIES`), including its two invariants: the + // sleep is clamped to the SHARED deadline (so a retry can never stretch + // `--timeout`), and the retry is per member — one throttled run does not + // stall the pool, because each lane runs this loop inside its own task. + let rateLimitAttempt = 0; + for (;;) { + // A member dequeued AFTER the shared deadline has passed must not be + // granted a fresh minimum poll window (with --max-concurrency 1 that + // would extend the invocation by ~1s per queued run past --timeout). + // Re-evaluated on every iteration so a retry obeys the same rule. + const remainingSeconds = Math.ceil((deadlineMs - Date.now()) / 1000); + if (remainingSeconds <= 0) return { kind: 'timeout' }; + try { + const run = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: remainingSeconds, + sleep: deps.sleep, + shutdown: shutdownOf(deps), + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + ticker.update(`Run ${run.runId} — ${run.status} (elapsed=${elapsed}s)`); + }, + resolveAlternate, + }); + return { kind: 'result', run }; + } catch (err) { + if (err instanceof TimeoutError) return { kind: 'timeout' }; + if (err instanceof RequestTimeoutError) throw err; + // Interrupt must reject the fan-out (handled at the collect point), not + // be flattened into a per-member 'error' outcome that would swallow the + // 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; + if (err instanceof ApiError) { + // `isTransientRateLimit` is the second gate, for an "unknown" 429 that + // carries neither a Retry-After nor the per-minute wording: treat it as + // permanent rather than burn the budget on it. (The credit-depletion + // 429 never reaches here at all — `errors.ts` re-maps that envelope to + // INSUFFICIENT_CREDITS before this catch sees it.) + if ( + err.code === 'RATE_LIMITED' && + isTransientRateLimit(err) && + rateLimitAttempt < WAIT_POLL_RATE_MAX_OUTER_RETRIES + ) { + rateLimitAttempt++; + const retryAfterMs = resolveRateLimitRetryMs(err); + const clampedRetryMs = Math.min(retryAfterMs, deadlineMs - Date.now()); + // Deadline already reached: the wait budget genuinely ran out, so the + // honest outcome is `timeout` — the same call the trigger fan-out + // makes ("Timed out … during rate-limit backoff"). Reporting the 429 + // instead would let the exit-11 escalation below claim "nothing else + // went wrong" for an invocation that in fact exhausted `--timeout`. + if (clampedRetryMs <= 0) return { kind: 'timeout' }; + stderrFn( + `[wait] ${runId} — rate limited (attempt ${rateLimitAttempt}/${WAIT_POLL_RATE_MAX_OUTER_RETRIES}): retrying in ${Math.ceil(clampedRetryMs / 1000)}s`, + ); + // Arm the graceful-detach scope across the backoff. `pollRunUntilTerminal` + // disarms in its own `finally`, so without this the sleep is a window + // where a first Ctrl-C hard-exits instead of taking the DEV-331 detach + // path (partial `{runId, status:'running'}` on stdout + honest hint). + const shutdown = shutdownOf(deps); + const disarm = shutdown.arm(); + try { + await sleepUntilOrInterrupt(clampedRetryMs, shutdown.signal, sleepFn); + } finally { + disarm(); + } + continue; + } + return { kind: 'error', code: err.code, exitCode: err.exitCode }; + } + return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; + } } }; @@ -5972,6 +6661,36 @@ export async function runTestWaitMany( authError.exitCode, ); } + // Rate limiting escalates the same way auth does, for the same reason: when it + // is the ONLY thing that went wrong, exit 7 ("timeout or per-member poll + // error") is actively misleading — nothing timed out and no run misbehaved, the + // client was throttled — and 7 tells an automated caller to re-attach + // immediately, which walks straight back into the limiter. Exit 11 says "back + // off, then re-attach", which is the correct action. Deliberately narrow: it + // requires that EVERY non-passed member is a rate-limited poll error, so a real + // timeout or a genuinely failed run is never masked (those keep 7 / 1). By this + // point each member has already spent its `WAIT_POLL_RATE_MAX_OUTER_RETRIES` + // Retry-After backoffs, so this is a persistently throttled window, not a blip. + // `outcomes` is keyed by runId, so a REPEATED id has one shared entry that the + // last lane to finish overwrites — a genuine `failed` can be replaced by a + // later RATE_LIMITED, which would make the counts below claim "nothing else + // went wrong" for an invocation that observed a failure. Rather than change the + // long-standing one-row-per-input-id output shape, the escalation simply + // declines to fire when the caller passed a duplicate; exit 7 is then the same + // answer as before this change. + const idsAreUnique = new Set(opts.runIds).size === opts.runIds.length; + const rateLimitedOnly = + idsAreUnique && + errors > 0 && + timedOut === 0 && + failed === 0 && + [...outcomes.values()].every(o => o.kind !== 'error' || o.code === 'RATE_LIMITED'); + if (rateLimitedOnly) { + throw new CLIError( + `Multi-run wait: rate limited on ${errors} of ${results.length} runs — back off and re-attach with: testsprite test wait ${unfinishedIds.join(' ')}`, + 11, + ); + } if (timedOut > 0 || errors > 0) { throw new CLIError( `Multi-run wait: ${timedOut} timed out, ${errors} poll error(s) out of ${results.length} runs`, @@ -6286,6 +7005,26 @@ export async function runTestWait( ); throw err; } + // RATE_LIMITED during polling — see the matching comment in runTestRun. + // The HTTP layer already retried internally and gave up; the run keeps + // executing (and billing) server-side, so this must emit the same + // partial-envelope + honest hint as the timeout/interrupt paths. The SAME + // ApiError is rethrown unchanged so its native exit code (11) is kept. + if (err instanceof ApiError && err.code === 'RATE_LIMITED') { + ticker.finalize(`Run ${opts.runId} — rate limited by the server`); + const partial = { runId: opts.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (rate limited by the server)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); + stderrFn(rateLimitedDetachMessage(err, [opts.runId])); + throw err; + } // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1) — see runTestRun. if (err instanceof InterruptError) { ticker.finalize(`Run ${opts.runId} — interrupted (${err.signal})`); @@ -7050,8 +7789,9 @@ interface CliRerunResult { /** * `test rerun` — M3.4 piece-3. * - * FE: `POST /tests/{id}/runs/rerun` → verbatim replay (no credit). With - * `--wait`, polls `GET /runs/{runId}` until terminal. + * FE: `POST /tests/{id}/runs/rerun` → verbatim replay, billed at 0.5 credits + * (same as a fresh run; legacy V2 accounts: free). With `--wait`, polls + * `GET /runs/{runId}` until terminal. * * BE: same route → closure + per-member runIds. With `--wait`, polls every * closure-member runId; exits on the named test's verdict; failed closure @@ -7237,7 +7977,10 @@ export async function runTestRerun( testId, { source: 'cli', - ...(effectiveAutoHeal ? { autoHeal: true } : {}), + // Always send the effective boolean, including an explicit `false` + // opt-out — the server defaults an ABSENT field to heal-on, so + // omitting the key on opt-out silently discarded --no-auto-heal. + autoHeal: effectiveAutoHeal, ...(opts.skipDependencies ? { skipDependencies: true } : {}), }, { idempotencyKey }, @@ -7282,8 +8025,11 @@ export async function runTestRerun( // Print auto-heal advisory. // CLI path: auto-heal is default-on for FE reruns (--no-auto-heal to opt // out). Free and paid CLI callers both get auto-heal; backend no longer - // tier-gates for source='cli'. Cost: 0.2 credits per engage (charged only - // when Phase-2 heal actually runs; verbatim replay passes are free). + // tier-gates for source='cli'. The rerun itself is billed at 0.5 credits + // regardless of whether heal engages (same as a fresh run; legacy V2 + // accounts: a verbatim replay pass is free). A heal engage costs an + // additional 0.2 credits on top of that, charged only when Phase-2 heal + // actually runs. // // Defensive branch: if the server still echoes autoHeal:false after we sent // autoHeal:true, the server did not apply it (unexpected; may happen on @@ -7297,16 +8043,14 @@ export async function runTestRerun( // default-on `true`, so opts.autoHeal && !rerunResp.autoHeal would // fire spuriously for every BE rerun). if (effectiveAutoHeal && !rerunResp.autoHeal) { - // Env-correct billing link (dev/prod portals differ); route-only when - // the API host is unknown. - const advisoryPortalBase = resolvePortalBase(resolveApiUrl(opts, deps)); + // Points at `testsprite usage`, not a hardcoded billing URL: this CLI + // path has no per-request org context (no backend nextAction feeds + // this advisory, and a personal-vs-org-bound key can't be told apart + // here), while `usage` already renders whichever wallet (personal or + // organization) actually governs this key's balance. stderrFn( `[advisory] auto-heal was not applied by the server (verbatim replay).` + - ` If this was unexpected, check your balance at ${ - advisoryPortalBase !== undefined - ? `${advisoryPortalBase}/dashboard/settings/billing` - : 'the portal Billing page (/dashboard/settings/billing)' - }.`, + ` If this was unexpected, check your balance with \`testsprite usage\`.`, ); } else if (rerunResp.autoHeal) { stderrFn( @@ -7314,6 +8058,11 @@ export async function runTestRerun( ); } + // Server advisory: the autoHeal opt-out was forwarded to the execution + // engine but is not yet enforced there. Present only on a V3-routed + // rerun with an explicit autoHeal:false request; absent everywhere else. + emitRerunAdvisories(stderrFn, rerunResp.advisories); + const isBERerun = !!rerunResp.closure; if (isBERerun && rerunResp.closure) { @@ -7367,9 +8116,24 @@ export async function runTestRerun( // BE rerun: poll every closure-member runId, exit on named test's verdict. const namedRunId = rerunResp.runId; const closureMembers = rerunResp.closure.members; - const closureFailures: Array<{ testId: string; runId: string; status: string }> = []; - - const pollMember = async (member: RerunClosureMember): Promise => { + // `unobserved`: member never reached a terminal verdict (timed out or its + // poll errored). Distinct from an observed non-passed run (failed/blocked). + const closureFailures: Array<{ + testId: string; + runId: string; + status: string; + unobserved?: boolean; + }> = []; + + // A member poll settles as one of these. Making the poll "total" (never + // throwing for a per-member failure) is what stops one member's error + // from rejecting the whole fan-out and discarding every sibling result. + type MemberOutcome = + | { kind: 'terminal'; run: RunResponse } + | { kind: 'timeout' } + | { kind: 'error'; status: string; error: unknown }; + + const pollMember = async (member: RerunClosureMember): Promise => { const resolveAlternate = makeBackendWaitFallback({ client, resolveTestId: () => member.testId, @@ -7377,7 +8141,7 @@ export async function runTestRerun( onResolved: () => undefined, }); try { - return await pollRunUntilTerminal(client, member.runId, { + const finalRun = await pollRunUntilTerminal(client, member.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, shutdown: shutdownOf(deps), @@ -7396,11 +8160,19 @@ export async function runTestRerun( }, resolveAlternate, }); + return { kind: 'terminal', run: finalRun }; } catch (err) { - if (err instanceof TimeoutError) { - return null; - } - throw err; + if (err instanceof TimeoutError) return { kind: 'timeout' }; + // Preserve the two intentional whole-fan-out aborts: each has a + // dedicated outer-catch branch (RequestTimeoutError → all-running + // partial + re-attach hints + exit 7; InterruptError → DEV-331 + // graceful detach). Re-throw so the fan-out's `.catch(reject)` fires. + if (err instanceof RequestTimeoutError || err instanceof InterruptError) throw err; + // Any other member error (ApiError, transient 5xx, malformed + // response) is classified per-member instead of rejecting the whole + // fan-out and discarding every already-collected sibling result. + const status = err instanceof ApiError ? err.code : 'error'; + return { kind: 'error', status, error: err }; } }; @@ -7410,6 +8182,10 @@ export async function runTestRerun( const concurrencyLimit = opts.maxConcurrency; let inFlight = 0; let memberIdx = 0; + // Set when the NAMED test's own poll errors (not a timeout). Re-thrown + // after the payload is printed so its real error/exit code is preserved + // without discarding the sibling results collected alongside it. + let namedPollError: unknown; try { await new Promise((resolve, reject) => { @@ -7418,29 +8194,53 @@ export async function runTestRerun( const member = members[memberIdx++]!; inFlight++; pollMember(member) - .then(result => { - memberResults.set(member.runId, result); - if (member.runId !== namedRunId) { - if (result === null) { - // Timed-out closure member: treat as incomplete/failed so - // the exit-code path fires exit 7 rather than silently - // succeeding with an unobserved member. + .then(outcome => { + if (outcome.kind === 'terminal') { + memberResults.set(member.runId, outcome.run); + if (member.runId !== namedRunId && outcome.run.status !== 'passed') { + closureFailures.push({ + testId: member.testId, + runId: member.runId, + status: outcome.run.status, + }); + stderrFn( + `⚠ closure member ${member.testId} (runId: ${member.runId}) finished with status: ${outcome.run.status}`, + ); + } + } else if (outcome.kind === 'timeout') { + // Timed-out closure member: treat as incomplete/failed so + // the exit-code path fires exit 7 rather than silently + // succeeding with an unobserved member. + memberResults.set(member.runId, null); + if (member.runId !== namedRunId) { closureFailures.push({ testId: member.testId, runId: member.runId, status: 'timeout', + unobserved: true, }); stderrFn( `⚠ closure member ${member.testId} (runId: ${member.runId}) timed out — rerun did not reach terminal within --timeout`, ); - } else if (result.status !== 'passed') { + } + } else { + // Classified per-member error — recorded, never aborts the + // fan-out. The named test's error is stashed for re-throw; + // sibling errors surface in closureFailures[]. + memberResults.set(member.runId, null); + if (member.runId === namedRunId) { + namedPollError = outcome.error; + } else { + // A poll error means we never saw a terminal verdict — the + // run may still be in flight — so it's unobserved too. closureFailures.push({ testId: member.testId, runId: member.runId, - status: result.status, + status: outcome.status, + unobserved: true, }); stderrFn( - `⚠ closure member ${member.testId} (runId: ${member.runId}) finished with status: ${result.status}`, + `⚠ closure member ${member.testId} (runId: ${member.runId}) could not be confirmed terminal — poll error: ${outcome.status}`, ); } } @@ -7485,6 +8285,34 @@ export async function runTestRerun( ); throw fanOutErr; } + // RATE_LIMITED from any member's poll (pollMember only swallows + // TimeoutError into a null return — see above; every other error, + // including a RATE_LIMITED ApiError, propagates through .catch(reject) + // exactly like RequestTimeoutError). Same partial shape as the + // timeout path so the caller always has every closure-member runId on + // stdout; the SAME thrown ApiError is rethrown unchanged so its + // native exit code (11) is preserved. + if (fanOutErr instanceof ApiError && fanOutErr.code === 'RATE_LIMITED') { + ticker.finalize(`Closure fan-out — rate limited by the server`); + const dispatchedRunIds = closureMembers.map(m => ({ + runId: m.runId, + testId: m.testId, + role: m.role, + status: 'running' as const, + })); + out.print({ runId: namedRunId, status: 'running', closure: dispatchedRunIds }, () => + dispatchedRunIds + .map(m => `${m.role.padEnd(9)} ${m.testId} (runId: ${m.runId}) — running`) + .join('\n'), + ); + stderrFn( + rateLimitedDetachMessage( + fanOutErr, + closureMembers.map(m => m.runId), + ), + ); + throw fanOutErr; + } // Graceful detach (DEV-331): same partial shape as the timeout path — // SIG-6 requires the partial to list ALL dispatched runIds. if (fanOutErr instanceof InterruptError) { @@ -7541,6 +8369,24 @@ export async function runTestRerun( }); if (!namedResult) { + // Named test's own poll errored (not a timeout): re-throw its real + // error now that the payload (incl. sibling closureFailures) is + // printed, preserving the true exit code instead of masking it as a + // timeout. + if (namedPollError !== undefined) { + // Every dispatched member is still executing (and billing), so a + // rate-limited named poll owes the same detach hints the fan-out's + // own branch emits — which this re-throw bypasses. + if (namedPollError instanceof ApiError && namedPollError.code === 'RATE_LIMITED') { + stderrFn( + rateLimitedDetachMessage( + namedPollError, + closureMembers.map(m => m.runId), + ), + ); + } + throw namedPollError; + } // timeout throw ApiError.fromEnvelope({ error: { @@ -7560,24 +8406,24 @@ export async function runTestRerun( throw new CLIError(`Run ${namedRunId} finished with status: ${namedResult.status}`, 1); } - // Fix B: timed-out closure members (non-named) are recorded in - // closureFailures with status 'timeout'. Even when the named run passes, - // we must exit 7 so --wait does not silently succeed when the closure - // as a whole was never observed to reach terminal. - const timedOutMembers = closureFailures.filter(f => f.status === 'timeout'); - if (timedOutMembers.length > 0) { - const timedOutIds = timedOutMembers.map(f => f.runId); + // Any unobserved member (timed out OR poll-errored) flips --wait to exit 7, + // even when the named run passes — otherwise a dependency whose status was + // never confirmed would let --wait exit 0. Observed-failed members are not + // included; that's the future --fail-on-closure decision. + const unobservedMembers = closureFailures.filter(f => f.unobserved); + if (unobservedMembers.length > 0) { + const unobservedIds = unobservedMembers.map(f => f.runId); const resumeHints = - timedOutIds.map(runId => `testsprite test wait ${runId}`).join('\n') + - `\nCancel instead: testsprite test cancel ${timedOutIds.join(' ')}`; + unobservedIds.map(runId => `testsprite test wait ${runId}`).join('\n') + + `\nCancel instead: testsprite test cancel ${unobservedIds.join(' ')}`; throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', - message: `${timedOutMembers.length} closure member${timedOutMembers.length !== 1 ? 's' : ''} timed out before reaching terminal status.`, + message: `${unobservedMembers.length} closure member${unobservedMembers.length !== 1 ? 's' : ''} did not reach an observed terminal status (timed out or errored during polling).`, nextAction: resumeHints, requestId: 'local', details: { - timedOutRunIds: timedOutMembers.map(f => f.runId), + unobservedRunIds: unobservedIds, timeoutSeconds: opts.timeoutSeconds, }, }, @@ -7651,6 +8497,24 @@ export async function runTestRerun( ); throw err; } + // RATE_LIMITED during polling — see the matching comment in runTestRun. + // Same partial-envelope contract; the SAME thrown ApiError is rethrown + // unchanged so its native exit code (11) is preserved. + if (err instanceof ApiError && err.code === 'RATE_LIMITED') { + ticker.finalize(`Run ${rerunResp.runId} — rate limited by the server`); + const partial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (rate limited by the server)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); + stderrFn(rateLimitedDetachMessage(err, [rerunResp.runId])); + throw err; + } // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1) — see runTestRun. if (err instanceof InterruptError) { ticker.finalize(`Run ${rerunResp.runId} — interrupted (${err.signal})`); @@ -7823,7 +8687,9 @@ export async function runTestRerun( { source: 'cli', testIds: chunk, - ...(effectiveAutoHeal ? { autoHeal: true } : {}), + // Always send the effective boolean, including an explicit `false` + // opt-out — see the matching comment on the single-rerun call site. + autoHeal: effectiveAutoHeal, ...(opts.skipDependencies ? { skipDependencies: true } : {}), }, { idempotencyKey: chunkKey }, @@ -7872,11 +8738,18 @@ export async function runTestRerun( byProject: mergeBatchRerunClosureByProject(chunkResponses.flatMap(r => r.closure.byProject)), }, notFound: chunkResponses.flatMap(r => r.notFound ?? []), + // Absent on every response except a V3-routed batch containing + // at least one FE test with an explicit autoHeal:false opt-out. Dedupe + // across chunks — every chunk in the same invocation carries the same + // autoHeal request, so the same advisory would otherwise repeat per chunk. + advisories: dedupeRerunAdvisories(chunkResponses.flatMap(r => r.advisories ?? [])), }; // Print dispatch summary // Mutable: D3 deferred-retry loop may append to `accepted`/`conflicts` and - // drain `deferred` under --wait. + // drain `deferred` under --wait. `advisories` may also grow if a D3 retry + // response carries an advisory the initial dispatch didn't (defensive — + // in practice the same request shape produces the same advisory set). let accepted = batchResp.accepted.slice(); let deferred = batchResp.deferred.slice(); let conflicts = batchResp.conflicts.slice(); @@ -7884,6 +8757,9 @@ export async function runTestRerun( // the retry window; the retry response's notFound[] is merged into this set so // the test is never reported as "resolved" when it actually vanished. let notFound = (batchResp.notFound ?? []).slice(); + // Mutable so a D3 deferred-retry response can contribute an + // advisory the initial dispatch didn't carry (defensive; see comment above). + let advisories = (batchResp.advisories ?? []).slice(); const closureByProject = batchResp.closure.byProject; const addedProducersTotal = closureByProject.reduce((n, p) => n + p.addedProducers.length, 0); @@ -7988,7 +8864,10 @@ export async function runTestRerun( { source: 'cli', testIds: chunk, - ...(effectiveAutoHeal ? { autoHeal: true } : {}), + // Always send the effective boolean, including an explicit + // `false` opt-out — see the matching comment on the initial + // dispatch call site above. + autoHeal: effectiveAutoHeal, ...(opts.skipDependencies ? { skipDependencies: true } : {}), }, { idempotencyKey: retryKey }, @@ -8011,6 +8890,12 @@ export async function runTestRerun( // into the running notFound set and remove from deferred so it isn't // reported as "resolved" in the final output. const newlyNotFound = retryChunkResponses.flatMap(r => r.notFound ?? []); + // Merge any advisories the retry response carries into the + // running set (deduped — see the initial-dispatch comment above). + const newlyAdvisories = retryChunkResponses.flatMap(r => r.advisories ?? []); + if (newlyAdvisories.length > 0) { + advisories = dedupeRerunAdvisories(advisories.concat(newlyAdvisories)); + } if (newlyDuplicateCount > 0) { stderrFn( @@ -8053,10 +8938,14 @@ export async function runTestRerun( } } + // Print the (deduped) advisory set once, after any D3 retries have + // had a chance to contribute one, not once per chunk/attempt. + emitRerunAdvisories(stderrFn, advisories); + if (!opts.wait) { // [P2] Build output from post-retry mutable state so deferred/conflicts/notFound // reflect what the D3 loop discovered, not just the initial batchResp. - out.print({ ...batchResp, accepted, deferred, conflicts, notFound }); + out.print({ ...batchResp, accepted, deferred, conflicts, notFound, advisories }); if (deferred.length > 0) { throw new CLIError( `Batch rerun incomplete: ${deferred.length} test${deferred.length !== 1 ? 's' : ''} were rate-deferred. Retry with: testsprite test rerun ${deferred.map(d => d.testId).join(' ')}`, @@ -8082,13 +8971,13 @@ export async function runTestRerun( }); } // [P2] Return post-retry state including merged notFound. - return { ...batchResp, accepted, deferred, conflicts, notFound }; + return { ...batchResp, accepted, deferred, conflicts, notFound, advisories }; } // --wait: fan-out poll each accepted run by its runId if (accepted.length === 0) { // [P2] Build output from post-retry mutable state including merged notFound. - out.print({ ...batchResp, accepted, deferred, conflicts, notFound }); + out.print({ ...batchResp, accepted, deferred, conflicts, notFound, advisories }); if (deferred.length > 0) { throw new CLIError( `Batch rerun: no tests were accepted (${deferred.length} deferred). ` + @@ -8114,7 +9003,7 @@ export async function runTestRerun( }); } // [P2] Return post-retry state including merged notFound. - return { ...batchResp, accepted, deferred, conflicts, notFound }; + return { ...batchResp, accepted, deferred, conflicts, notFound, advisories }; } const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); @@ -8271,6 +9160,9 @@ export async function runTestRerun( // report the partial run as fully successful. Mirrors the non-wait // `out.print(batchResp)` path. notFound, + // Mirrors the non-wait `out.print(batchResp)` path — carry the + // (deduped, post-retry) advisory set into the --wait JSON payload too. + advisories, closure: batchResp.closure, summary: { passed, @@ -8338,7 +9230,7 @@ export async function runTestRerun( // final accounting (accepted = original BatchRerunAccepted[] dispatch list // as required by the BatchRerunResponse type; rerunResults is the polled // outcome printed to stdout and is not part of the returned shape). - return { ...batchResp, accepted, deferred, conflicts, notFound }; + return { ...batchResp, accepted, deferred, conflicts, notFound, advisories }; } // --------------------------------------------------------------------------- @@ -8662,6 +9554,12 @@ export function createTestCommand(deps: TestDeps = {}): Command { 'JSON file with the full FE test definition — projectId, type, name, planSteps[] all live in the file ' + '(≤ 256 KB; mutually exclusive with --code-file). In this mode --project/--type/--name/--description/--priority are ignored.', ) + .option( + '--plan-template', + 'print a minimal valid plan-file skeleton to stdout and exit (pure-local: no network, no credentials, ' + + 'ignores every other flag). Pipe to a file and edit: `--plan-template > plan.json`.', + false, + ) .option( '--run', 'after create, trigger the test. Combine with --wait to block until terminal.', @@ -8690,6 +9588,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--category ', "BE only: test category. Use 'teardown' or 'cleanup' to mark a final-wave cleanup test.", ) + .addHelpText('after', PLAN_TEMPLATE_HELP_TEXT) .addHelpText( 'after', '\nBE dependency authoring (M4):\n' + @@ -8699,6 +9598,13 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (cmdOpts: CreateFlagOpts, command: Command) => { + // Pure-local, no network/credentials, no other flag is + // consulted. Checked first so `--plan-template` never trips the + // --plan-from/--code-file mutual-exclusivity guard below. + if (cmdOpts.planTemplate === true) { + runPlanTemplate(resolveCommonOptions(command), deps); + return; + } // --plan-from and --code-file are mutually exclusive. Dispatch // here so each `run*` function stays single-purpose. If neither // is set, the existing runCreate path enforces --code-file. @@ -8764,8 +9670,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { name: cmdOpts.name, description: cmdOpts.description, priority: parseEnumFlag(cmdOpts.priority, 'priority', CLI_CREATE_PRIORITIES) as - | CliCreatePriority - | undefined, + CliCreatePriority | undefined, codeFile: cmdOpts.codeFile, idempotencyKey: cmdOpts.idempotencyKey, // M3.3 chain flags: @@ -8961,6 +9866,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option('--page-size ', 'with --history: number of runs per page (1–100, default 20)') .option('--cursor ', 'with --history: opaque cursor from a prior page') + .option('--rerun', 'with --history: show only reruns') + .option('--no-rerun', 'with --history: show only fresh (non-rerun) runs') .option('--columns ', 'with --history: select/reorder text table columns') .option('--no-header', 'with --history: suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) @@ -8978,6 +9885,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ? parseNumericFlag(cmdOpts.pageSize, 'page-size') : undefined, cursor: cmdOpts.cursor, + rerun: cmdOpts.rerun, columns: cmdOpts.columns, noHeader: cmdOpts.header === false, }, @@ -9031,8 +9939,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { name: cmdOpts.name, description: cmdOpts.description, priority: parseEnumFlag(cmdOpts.priority, 'priority', CLI_CREATE_PRIORITIES) as - | CliCreatePriority - | undefined, + CliCreatePriority | undefined, produces: cmdOpts.produces, needs: cmdOpts.needs, category: cmdOpts.category, @@ -9313,6 +10220,11 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' is recorded as error: in its row and folded into exit 7)\n' + ' 7 timeout or per-member poll error — resume with: testsprite test wait \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + + ' 11 rate limited, and nothing else went wrong — polls are retried\n' + + ' automatically honoring Retry-After first, so this means the throttle\n' + + ' outlasted that budget while the runs were still fine. Back off, then\n' + + ' re-attach with test wait. (A throttle that instead consumes the whole\n' + + ' --timeout reports 7, and any real timeout or failure keeps 7 / 1.)\n' + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + '\nCtrl-C detaches only (the run keeps executing and billing); stop it for\n' + 'real with: testsprite test cancel ', @@ -9359,7 +10271,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { test .command('rerun [test-ids...]') .description( - 'Re-execute a test (or multiple) as a cheap replay — FE replays the saved script (no credit), BE re-runs the dependency closure.\n' + + 'Re-execute a test (or multiple) as a replay — FE replays the saved script, BE re-runs the dependency closure. ' + + 'Billed the same as a fresh run: 0.5 credits per FE rerun / 0.2 credits per BE rerun (legacy V2 accounts: FE rerun remains free).\n' + '\nExit codes:\n' + ' 0 passed (or queued without --wait)\n' + ' 1 failed / blocked / cancelled\n' + @@ -9432,7 +10345,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' • Under --wait the per-request HTTP timeout is auto-raised to cover --timeout so a\n' + ' slow trigger/poll under load is not cut at the 120s default (see --request-timeout).\n' + ' • Batch --wait: rate-deferred tests appear in `deferred[]` and `summary.deferred`,\n' + - ' and force a non-zero exit — they are NOT counted in `summary.total` (dispatched only).', + ' and force a non-zero exit — they are NOT counted in `summary.total` (dispatched only).\n' + + ' • On V3-routed accounts, --no-auto-heal is still rolling out and may not yet be\n' + + ' honored server-side — check `auth status` for your routing.', ) .addHelpText( 'after', @@ -9516,8 +10431,11 @@ export function createTestCommand(deps: TestDeps = {}): Command { .addHelpText( 'after', '\nNotes:\n' + - ' • Frontend replays are free verbatim script replays (no credit); backend replays\n' + - ' re-run the dependency closure and may cost credits — a one-line advisory is printed.\n' + + ' • Each replay is billed as a rerun — 0.5 credits for a frontend replay (verbatim\n' + + ' script), 0.2 credits for a backend replay (re-runs the dependency closure) —\n' + + ' same price as a fresh run, so `--runs N` costs roughly N×0.5 credits for a\n' + + ' frontend test (legacy V2 accounts: FE rerun remains free). A one-line advisory\n' + + ' is printed before a backend replay.\n' + ' • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass;\n' + ' this measures replay stability of the saved script against the configured URL.\n' + ' • `--output json` emits a machine-readable stability report for CI gating.', @@ -9555,7 +10473,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { // `test flaky` — repeat-run flaky-test detector // --------------------------------------------------------------------------- -/** Upper bound on `--runs` so a repeat-runner can't amplify free FE replays. */ +/** Upper bound on `--runs` so a repeat-runner can't rack up unbounded rerun charges. */ const MAX_FLAKY_RUNS = 10; /** Default replay count when `--runs` is omitted. */ const DEFAULT_FLAKY_RUNS = 5; @@ -9586,9 +10504,10 @@ interface RunTestFlakyOptions extends CommonOptions { * `test flaky ` — replay a test N times and report a stability score. * * Each attempt is a `POST /tests/{id}/runs/rerun` with auto-heal OFF (a strict - * verbatim replay) followed by `pollRunUntilTerminal`. Frontend replays are - * free verbatim script replays; backend replays re-run the dependency closure - * (a one-line credit advisory is printed). The pure scoring lives in + * verbatim replay) followed by `pollRunUntilTerminal`. Each replay is billed + * as a rerun (0.5 credits FE / 0.2 credits BE, same as a fresh run; legacy V2 + * accounts: FE rerun remains free) — a one-line credit advisory is printed + * for backend tests. The pure scoring lives in * `lib/flaky.ts`; this function is the I/O orchestrator. * * Exit code: 0 when every observed attempt passed (stable), else 1 — so CI can @@ -9641,12 +10560,16 @@ export async function runFlaky( if (isBackend) { stderrFn( `[advisory] ${opts.testId} is a backend test — each replay re-runs its dependency closure ` + - `and may cost credits. Frontend replays are free verbatim script replays; backend replays are not.`, + `and is billed at 0.2 credits per rerun, same as a fresh run (frontend reruns are 0.5 credits).`, ); } const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); const attempts: FlakyAttempt[] = []; + // Collected across attempts and deduped — same request shape every + // attempt, so the server would otherwise repeat the identical advisory once + // per replay. Surfaced ONCE in the final report/summary, not N times. + let advisories: RerunAdvisory[] = []; for (let i = 1; i <= opts.runs; i++) { const idempotencyKey = `cli-flaky-${randomUUID()}`; @@ -9654,8 +10577,16 @@ export async function runFlaky( let rerunResp: RerunResponse; try { // auto-heal is intentionally OFF: flaky detection needs a strict verbatim - // replay so healed drift cannot mask a nondeterministic pass/fail. - rerunResp = await client.triggerRerun(opts.testId, { source: 'cli' }, { idempotencyKey }); + // replay so healed drift cannot mask a nondeterministic pass/fail. Sent + // explicitly (not omitted) — an absent field defaults to heal-on server-side. + rerunResp = await client.triggerRerun( + opts.testId, + { source: 'cli', autoHeal: false }, + { idempotencyKey }, + ); + if (rerunResp.advisories && rerunResp.advisories.length > 0) { + advisories = dedupeRerunAdvisories(advisories.concat(rerunResp.advisories)); + } } catch (err) { // A missing replayable run is fatal for the whole command (mirror rerun): // there is nothing to repeat, so point the user at a fresh `test run`. @@ -9711,6 +10642,16 @@ export async function runFlaky( stderrFn(interruptDetachMessage(err, [runId])); throw err; } + // RATE_LIMITED during polling (same defect class as the InterruptError + // branch above): the HTTP layer already retried and gave up, but this + // attempt's run keeps executing (and billing) server-side. Name it on + // stderr before rethrowing so the runId is never silently dropped; + // rethrow the SAME ApiError unchanged so its exit code (11) is kept. + if (err instanceof ApiError && err.code === 'RATE_LIMITED') { + ticker.finalize(`Attempt ${i}/${opts.runs} — rate limited by the server`); + stderrFn(rateLimitedDetachMessage(err, [runId])); + throw err; + } // A per-attempt deadline (poll TimeoutError) or a client-side request // timeout both count as a non-passing "timeout" outcome for this attempt. if (err instanceof TimeoutError || err instanceof RequestTimeoutError) { @@ -9729,7 +10670,11 @@ export async function runFlaky( ticker.finalize(); - const report = summarizeFlaky(opts.testId, attempts); + // Print the deduped advisory set once for the whole probe, not + // once per replay attempt (mirrors the `test rerun` advisory rendering). + emitRerunAdvisories(stderrFn, advisories); + + const report = summarizeFlaky(opts.testId, attempts, advisories); out.print(report, data => renderFlakyText(data as FlakyReport)); const exitCode = flakyExitCode(report); @@ -9815,6 +10760,8 @@ interface ResultFlagOpts { pageSize?: string; /** Opaque pagination cursor from a prior page's nextCursor. */ cursor?: string; + /** Filter history by rerun-ness: --rerun (only reruns) / --no-rerun (only fresh). */ + rerun?: boolean; columns?: string; header?: boolean; } @@ -9825,6 +10772,8 @@ interface CreateFlagOpts { name: string; description?: string; planFrom?: string; + /** Print the canonical plan-file skeleton and exit. */ + planTemplate?: boolean; run?: boolean; wait?: boolean; timeout?: string; @@ -10461,7 +11410,7 @@ function plannedBundleFiles(ctx: CliFailureContext, failedOnly: boolean): string files.push('result.json'); files.push('failure.json'); files.push(`code.${pickCodeExtension(ctx.code.language, ctx.code.framework)}`); - if (ctx.result.videoUrl) files.push('video.mp4'); + if (ctx.result.videoUrl) files.push(`video.${pickVideoExtension(ctx.result.videoUrl)}`); const stepsToInclude = failedOnly ? ctx.steps.filter(s => { @@ -10737,8 +11686,7 @@ function createTestCodeCommand(deps: TestDeps): Command { expectedVersion: cmdOpts.expectedVersion, force: cmdOpts.force === true, language: parseEnumFlag(cmdOpts.language, 'language', CODE_PUT_LANGUAGES) as - | CodePutLanguage - | undefined, + CodePutLanguage | undefined, idempotencyKey: cmdOpts.idempotencyKey, dryRunSimulateError: simulateError === 'PRECONDITION_FAILED' ? 'PRECONDITION_FAILED' : undefined, diff --git a/src/commands/test.wait.spec.ts b/src/commands/test.wait.spec.ts index 4830e50..c0eeec1 100644 --- a/src/commands/test.wait.spec.ts +++ b/src/commands/test.wait.spec.ts @@ -796,6 +796,44 @@ describe('runTestWait — backend testId fallback (L1888)', () => { expect(stderr.join(' ')).toContain('test record'); }); + it('backend fallback with no URL on either side keeps targetUrl null and omits the text line', async () => { + const { credentialsPath } = makeCreds(); + // Both the run row and the test record report no target URL — the shape a + // backend run has (no browser URL to record). + const router = beWaitRouter({ + runStatus: 'running', + result: () => makeBeResult({ status: 'passed', targetUrl: null, codeVersion: null }), + }); + const stdoutLines: string[] = []; + const result = await runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch((url: string) => + url.includes('/runs/run_abc') + ? { body: { ...makeRun('running'), targetUrl: null, codeVersion: null } } + : router.handler(url), + ), + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + expect(result.status).toBe('passed'); + expect(result.targetUrl).toBeNull(); + const stdoutBlock = stdoutLines.join('\n'); + // No dangling "targetUrl" label with an empty value, and no literal "null". + expect(stdoutBlock).not.toMatch(/^targetUrl/m); + expect(stdoutBlock).not.toContain('null'); + }); + it('failing backend test resolves via fallback (CLIError exit 1) + testId artifact hint', async () => { const { credentialsPath } = makeCreds(); const router = beWaitRouter({ @@ -1022,6 +1060,108 @@ describe('runTestWait: Fix 3 — RequestTimeoutError writes partial JSON to stdo }); }); +// --------------------------------------------------------------------------- +// RATE_LIMITED during test wait polling — partial stdout + honest hint, exit +// 11 kept (never reclassified to 7). The 429 must be a real HTTP response +// (not thrown directly) so it exercises http.ts's own RATE_LIMITED +// retry-then-throw path (MAX_ATTEMPTS_RATE_LIMITED = 3). +// --------------------------------------------------------------------------- + +function rateLimitedResponse(retryAfterSeconds?: number): Response { + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Run trigger rate limit exceeded: too many requests from this IP.', + nextAction: '', + requestId: 'req_rl_wait_test', + details: {}, + }, + }), + { + status: 429, + headers: { + 'content-type': 'application/json', + ...(retryAfterSeconds !== undefined ? { 'retry-after': String(retryAfterSeconds) } : {}), + }, + }, + ); +} + +describe('runTestWait: RATE_LIMITED writes partial JSON to stdout, keeps exit 11', () => { + it('exit 11 (NOT reclassified to 7) AND stdout contains {runId, status:"running"} when poll exhausts RATE_LIMITED retries', async () => { + const { credentialsPath } = makeCreds(); + // retry-after: 0 keeps http.ts's internal retry-then-throw budget fast. + const fetchImpl: typeof globalThis.fetch = async () => rateLimitedResponse(0); + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const err = await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ).catch(e => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('RATE_LIMITED'); + expect((err as ApiError).exitCode).toBe(11); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('run_abc'); + expect(stderrBlock).toContain('test wait'); + expect(stderrBlock).toContain('test cancel'); + expect(stderrBlock).toContain('Rate limited'); + }); + + it('text mode: renders human-readable partial (not raw JSON), still exit 11', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl: typeof globalThis.fetch = async () => rateLimitedResponse(0); + + const stdoutLines: string[] = []; + + const err = await runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ).catch(e => e); + + expect((err as ApiError).exitCode).toBe(11); + const stdoutBlock = stdoutLines.join('\n'); + expect(stdoutBlock).toContain('runId'); + expect(stdoutBlock).toContain('running'); + expect(stdoutBlock).not.toMatch(/^\{/); + }); +}); + // --------------------------------------------------------------------------- // TimeoutError on test wait: partial stdout + exit 7 // --------------------------------------------------------------------------- @@ -1197,6 +1337,76 @@ describe('[fix-4-ux] runTestWait — text mode shows error string for failed/blo expect(stdoutBlock).not.toMatch(/^error\s+/m); }); + it('run with targetUrl: null completes normally and omits the targetUrl line in text mode', async () => { + const { credentialsPath } = makeCreds(); + // A backend run (and every V3-executed run) reports no target URL; those + // rows also report no codeVersion when the test has no stored code body. + const run: RunResponse = { ...makeRun('passed'), targetUrl: null, codeVersion: null }; + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(() => ({ body: run })); + + const result = await runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + + // Resolves (exit 0) instead of raising a response-validation INTERNAL. + expect(result.status).toBe('passed'); + const stdoutBlock = stdoutLines.join('\n'); + expect(stdoutBlock).toMatch(/^status\s+passed$/m); + // Absent fields are omitted, never printed as the literal "null". + expect(stdoutBlock).not.toMatch(/^targetUrl/m); + expect(stdoutBlock).not.toMatch(/^codeVersion/m); + expect(stdoutBlock).not.toContain('null'); + expect(stderrLines.join('\n')).not.toContain('INTERNAL'); + }); + + it('run with targetUrl: null passes the null through the JSON envelope', async () => { + const { credentialsPath } = makeCreds(); + const run: RunResponse = { ...makeRun('passed'), targetUrl: null, codeVersion: null }; + const stdoutLines: string[] = []; + + const fetchImpl = makeFetch(() => ({ body: run })); + + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + + const parsed = JSON.parse(stdoutLines.join('')) as RunResponse; + expect(parsed.status).toBe('passed'); + expect(parsed.targetUrl).toBeNull(); + expect(parsed.codeVersion).toBeNull(); + }); + it('JSON mode does NOT change: error field passes through wire envelope unchanged', async () => { const { credentialsPath } = makeCreds(); const run: RunResponse = { @@ -1305,6 +1515,81 @@ describe('runTestWait — dashboardUrl on terminal output', () => { const printed = JSON.parse(stdout.join('')) as Record; expect(printed.dashboardUrl).toBeUndefined(); }); + + // A server-sent link supersedes the client computation, because the server + // knows which STORE answered the read and the client does not: a V3-served + // run's page is a different route family, and the `/dashboard/tests/…` route + // the client templates reads DynamoDB only — for a V3-native project there is + // no row there at all, so the client's link cannot render. + it('server-sent dashboardUrl wins over the client computation', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const serverUrl = + 'https://www.testsprite.com/dashboard-v3/o/org-1/projects/project_1/test-cases/test_xyz'; + const fetchImpl = makeFetch(() => ({ + body: { ...makeRun('passed'), dashboardUrl: serverUrl }, + })); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.dashboardUrl).toBe(serverUrl); + }); + + // An explicit `null` is the server saying "no correct link exists for this + // run" (e.g. the workspace-scoped page isn't served by this environment's + // portal build yet). Falling back to the client guess there would put back + // exactly the broken link the server declined to send. + it('server dashboardUrl:null suppresses the link — the client guess is NOT substituted', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const fetchImpl = makeFetch(() => ({ body: { ...makeRun('passed'), dashboardUrl: null } })); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + const printed = JSON.parse(stdout.join('')) as Record; + expect(printed.dashboardUrl).toBeUndefined(); + }); + + it('text mode: a server link renders on the card in place of the client shape', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const serverUrl = + 'https://www.testsprite.com/dashboard-v3/o/org-1/projects/project_1/test-cases/test_xyz'; + const fetchImpl = makeFetch(() => ({ + body: { ...makeRun('passed'), dashboardUrl: serverUrl }, + })); + const stdout: string[] = []; + await runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { credentialsPath, fetchImpl, stdout: line => stdout.push(line), sleep: instantSleep }, + ); + const out = stdout.join('\n'); + expect(out).toContain(`dashboard ${serverUrl}`); + expect(out).not.toContain('/dashboard/tests/'); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/usage.test.ts b/src/commands/usage.test.ts index 28bdef3..71fd4b1 100644 --- a/src/commands/usage.test.ts +++ b/src/commands/usage.test.ts @@ -64,7 +64,7 @@ beforeEach(() => { }); describe('runUsage — dry-run', () => { - it('emits the dry-run banner + note about missing backend data', async () => { + it('emits the dry-run banner + note that these are sample values', async () => { const { capture, deps } = makeCapture(); const result = await runUsage( { profile: 'default', output: 'text', debug: false, dryRun: true }, @@ -73,8 +73,8 @@ describe('runUsage — dry-run', () => { const stderr = capture.stderr.join('\n'); // Banner must be present. expect(stderr).toContain('dry-run'); - // Must note that credits require a backend update. - expect(stderr).toContain('backend'); + // Must note that these are canned sample values, not a real balance. + expect(stderr).toContain('sample'); // Must return the canned sample. expect(result).toEqual(DRY_RUN_USAGE_SAMPLE); }); @@ -97,7 +97,7 @@ describe('runUsage — dry-run', () => { describe('runUsage — real path without credits (current backend)', () => { it('returns the /me response and emits a note about missing balance', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const result = await runUsage( { profile: 'default', output: 'text', debug: false }, @@ -116,7 +116,7 @@ describe('runUsage — real path without credits (current backend)', () => { }); it('text output includes identity fields even without credits', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runUsage( { profile: 'default', output: 'text', debug: false }, @@ -132,7 +132,7 @@ describe('runUsage — real path without credits (current backend)', () => { }); it('JSON output passes the raw /me response through (no credits key present)', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runUsage( { profile: 'default', output: 'json', debug: false }, @@ -150,7 +150,7 @@ describe('runUsage — real path without credits (current backend)', () => { describe('runUsage — real path with credits (future backend)', () => { it('renders balance block when credits + subPlan are present', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runUsage( { profile: 'default', output: 'text', debug: false }, @@ -173,7 +173,7 @@ describe('runUsage — real path with credits (future backend)', () => { }); it('does NOT emit the missing-balance note when credits are present', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runUsage( { profile: 'default', output: 'text', debug: false }, @@ -189,7 +189,7 @@ describe('runUsage — real path with credits (future backend)', () => { }); it('emits low-balance warning when credits < creditsPerRun', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const lowBalance: UsageResponse = { ...meWithCredits, credits: 1, creditsPerRun: 2 }; await runUsage( @@ -206,7 +206,7 @@ describe('runUsage — real path with credits (future backend)', () => { }); it('emits free-plan upgrade hint when subPlan is Free', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); const freePlan: UsageResponse = { ...meWithCredits, subPlan: 'Free', credits: 10 }; await runUsage( @@ -223,7 +223,7 @@ describe('runUsage — real path with credits (future backend)', () => { }); it('JSON output passes credits and subPlan through verbatim', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); await runUsage( { profile: 'default', output: 'json', debug: false }, @@ -240,6 +240,340 @@ describe('runUsage — real path with credits (future backend)', () => { }); }); +describe('runUsage — org wallet (activeOrg)', () => { + const meWithOrg: UsageResponse = { + ...meWithCredits, + // The org block renders only for V3-routed callers: the renderer checks + // the authoritative `v3Enabled` routing bit alongside `activeOrg`, + // never field presence alone. + v3Enabled: true, + activeOrg: { + id: 'org-1', + name: 'Acme QA', + plan: 'Standard', + role: 'admin', + remaining: 1650, + includedCredits: 1600, + seats: 3, + }, + }; + + it('renders the org block and suppresses the legacy balance block', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(meWithOrg) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('organization'); + expect(out).toContain('Acme QA (admin)'); + expect(out).toContain('Standard'); + // `balance:` (not `credits:`) — the legacy per-user number keeps the + // `credits` name in JSON output, so the org wallet must not reuse it. + expect(out).toContain('balance: 1650 remaining'); + expect(out).toContain('seats:'); + // Legacy per-user block is superseded — its lines must not render. + expect(out).not.toContain('cost per frontend run:'); + expect(out).not.toContain('can trigger:'); + }); + + it('low org balance triggers the top-up warning', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const drained: UsageResponse = { + ...meWithOrg, + activeOrg: { ...meWithOrg.activeOrg!, remaining: 1 }, + }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(drained) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('warning'); + expect(out).toContain('billing'); + }); + + // The org wallet is a DIFFERENT settings page than the personal + // `/dashboard/settings/billing` route (that page only manages the legacy + // per-user balance) — the low-balance warning must not point an org-bound + // caller at a page that can't actually top up their wallet. + it('low org balance warning does not point at the personal billing URL', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const drained: UsageResponse = { + ...meWithOrg, + activeOrg: { ...meWithOrg.activeOrg!, remaining: 1 }, + }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(drained) }, + ); + const out = capture.stdout.join('\n'); + expect(out).not.toContain('/dashboard/settings/billing'); + expect(out.toLowerCase()).toContain('org admin'); + }); + + // The non-org (legacy) low-balance path is unchanged: it still points at + // the real personal billing URL. + it('low personal balance warning still points at /dashboard/settings/billing', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const drainedLegacy: UsageResponse = { + ...meWithCredits, + credits: 0, + creditsPerRun: 2, + }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(drainedLegacy) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('/dashboard/settings/billing'); + }); + + it('Free org plan triggers the upgrade hint even when legacy subPlan is paid', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const freeOrg: UsageResponse = { + ...meWithOrg, + subPlan: 'Standard', // stale legacy value — org plan must win + activeOrg: { ...meWithOrg.activeOrg!, plan: 'Free', remaining: 150 }, + }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(freeOrg) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('Free plan'); + expect(out).toContain('pricing'); + }); + + it('JSON output passes activeOrg through verbatim', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'json', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(meWithOrg) }, + ); + const parsed = JSON.parse(capture.stdout.join('')) as UsageResponse; + expect(parsed.activeOrg).toEqual(meWithOrg.activeOrg); + }); + + it('absent activeOrg keeps the legacy rendering intact (older backends)', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(meWithCredits) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('cost per frontend run:'); + expect(out).not.toContain('organization'); + }); + + it('org account without legacy credits → org block renders and NO missing-balance note', async () => { + // A V3-native org account has no DDB user row, so the legacy `credits` + // field is legitimately absent — but its balance already rendered in the + // organization block, so the stderr "balance not returned" note must not + // fire (it would contradict the output right above it). + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const orgOnly: UsageResponse = { + ...meWithOrg, + credits: undefined, + subPlan: undefined, + creditsPerRun: undefined, + }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(orgOnly) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('organization'); + expect(out).toContain('1650 remaining'); + expect(capture.stderr.join('\n')).not.toContain('credit balance not returned'); + }); + + it('activeOrg present but v3Enabled false → legacy rendering (wallet selection follows routing, not field presence)', async () => { + // Defense-in-depth: a V2-routed caller's billable commands charge the + // legacy wallet, so an org block must never supersede the legacy lines + // for them — even if a backend regression ships activeOrg to such a + // caller again. The current backend never produces this combination. + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const v2Routed: UsageResponse = { ...meWithOrg, v3Enabled: false }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(v2Routed) }, + ); + const out = capture.stdout.join('\n'); + expect(out).not.toContain('organization'); + expect(out).toContain('cost per frontend run:'); + }); +}); + +describe('runUsage — org-bound key with degraded /me enrichment', () => { + // A Postgres-backed membership key (`sk-member-…`) — `cli.guard.ts` only ever + // populates `/me`'s `org` field for THIS key family; a legacy `sk-user-…` + // key never carries an org binding at all. Using a legacy key here would + // model a `{apiKey: sk-user-…, me: {org: {...}}}` combination the real + // backend can never produce, which would silently stop protecting the + // behavior these tests exist for. (`runUsage` itself doesn't branch on the + // key's own shape — only on what `/me` returns — so this only matters for + // fixture realism, not for making the assertions below pass.) + const MEMBERSHIP_KEY = `tsp_u_${'A'.repeat(43)}`; + + // `me.controller.ts` keeps `org` (no I/O — the key's own binding) even when + // the best-effort `activeOrg` Postgres enrichment throws and is swallowed. + // A V3-native org member has no legacy DynamoDB user row either, so + // `credits`/`subPlan` are absent too. This is the exact reachable state the + // backend's own tests cover (PG down + no DDB row → both `activeOrg` and + // `credits` missing) — org-boundedness must be read from `org`, never from + // `activeOrg`/`v3Enabled` alone, or this state falls through to the + // personal-billing branches. + const degradedOrgBound: UsageResponse = { + ...meWithoutCredits, + v3Enabled: true, + org: { id: 'org-1', name: 'Acme QA', role: 'admin' }, + // activeOrg, credits, subPlan all absent — enrichment degraded. + }; + + it('does not point the caller at the personal billing page', async () => { + writeProfile('default', { apiKey: MEMBERSHIP_KEY }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(degradedOrgBound) }, + ); + const out = capture.stdout.join('\n'); + const err = capture.stderr.join('\n'); + expect(out).not.toContain('/dashboard/settings/billing'); + expect(err).not.toContain('/dashboard/settings/billing'); + }); + + it('says the org balance could not be loaded, and asks for an org admin — not a fabricated org URL', async () => { + writeProfile('default', { apiKey: MEMBERSHIP_KEY }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(degradedOrgBound) }, + ); + const out = capture.stdout.join('\n'); + const err = capture.stderr.join('\n'); + const combined = `${out}\n${err}`; + expect(combined.toLowerCase()).toContain('could not be loaded'); + expect(combined.toLowerCase()).toContain('org admin'); + // No fabricated org-scoped settings URL — the CLI has no confirmed route. + expect(combined).not.toMatch(/https?:\/\/\S+\/dashboard/); + }); + + it('never falls back to rendering a legacy credits/plan block', async () => { + writeProfile('default', { apiKey: MEMBERSHIP_KEY }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(degradedOrgBound) }, + ); + const out = capture.stdout.join('\n'); + expect(out).not.toContain('--- credits & plan ---'); + expect(out).not.toContain('cost per frontend run:'); + }); + + // A user who signed up before V3 (and so still carries a legacy DynamoDB + // `credits`/`subPlan` row) can LATER be issued a membership key — `/me`'s + // billing enrichment GetItems `UserEntity` independently of the org read, + // so both can coexist. This is the fixture that actually exercises the + // `!isOrgBound` guards on the legacy block / low-balance / free-plan + // branches: `degradedOrgBound` above has no legacy fields at all, so those + // branches were already false before the guard existed and the assertions + // above passed vacuously. Here `credits`/`subPlan` are deliberately set to + // values that WOULD trip both the low-balance warning and the Free-plan + // upgrade hint on the personal branch if the org binding weren't checked + // first. + const migratedOrgBound: UsageResponse = { + ...meWithoutCredits, + v3Enabled: true, + org: { id: 'org-1', name: 'Acme QA', role: 'admin' }, + credits: 0, + subPlan: 'Free', + creditsPerRun: 2, + // activeOrg absent — degraded enrichment, same as above. + }; + + it('migrated user (stale legacy credits/subPlan + org binding): renders neither the legacy block nor its low-balance/Free-plan hints', async () => { + writeProfile('default', { apiKey: MEMBERSHIP_KEY }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(migratedOrgBound) }, + ); + const out = capture.stdout.join('\n'); + const err = capture.stderr.join('\n'); + expect(out).not.toContain('--- credits & plan ---'); + // Would have fired from the personal low-balance branch (credits:0 < + // creditsPerRun:2) had `!isOrgBound` not gated it. + expect(out).not.toContain('credit balance is below the per-run cost'); + // Would have fired from the personal Free-plan branch (subPlan:'Free') + // had `!isOrgBound` not gated it. + expect(out).not.toContain('Free plan'); + expect(out).not.toContain('pricing'); + expect(out).not.toContain('/dashboard/settings/billing'); + expect(err).not.toContain('/dashboard/settings/billing'); + // The honest degraded-org line still renders instead. + expect(out.toLowerCase()).toContain('could not be loaded'); + }); +}); + +describe('runUsage — org attribution', () => { + it('renders `orgs:` and `org binding:` lines when the backend supplies them', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meWithOrgs: UsageResponse = { + ...meWithoutCredits, + organizations: [{ id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }], + org: { id: 'org_1', name: 'Acme Corp', role: 'owner' }, + }; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(meWithOrgs) }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('orgs: Acme Corp (org_1, role: owner)'); + expect(out).toContain('org binding: Acme Corp (org_1, role: owner)'); + }); + + it('omits the org lines entirely when the backend does not return them', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(meWithoutCredits) }, + ); + const out = capture.stdout.join('\n'); + expect(out).not.toContain('orgs:'); + expect(out).not.toContain('org binding:'); + expect(out).not.toContain('undefined'); + }); + + it('--output json passes organizations[] and org through verbatim', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meWithOrgs: UsageResponse = { + ...meWithoutCredits, + organizations: [{ id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }], + org: { id: 'org_1', name: 'Acme Corp', role: 'owner' }, + }; + await runUsage( + { profile: 'default', output: 'json', debug: false }, + { ...deps, credentialsPath, fetchImpl: makeFetch(meWithOrgs) }, + ); + const parsed = JSON.parse(capture.stdout.join('')) as UsageResponse; + expect(parsed.organizations).toEqual(meWithOrgs.organizations); + expect(parsed.org).toEqual(meWithOrgs.org); + }); +}); + describe('runUsage — error handling', () => { it('throws AUTH_REQUIRED when no profile is configured', async () => { const { deps } = makeCapture(); @@ -249,7 +583,7 @@ describe('runUsage — error handling', () => { }); it('forwards server AUTH_INVALID with exit code 3', async () => { - writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-bad' }, { path: credentialsPath }); const { deps } = makeCapture(); const errorBody = { error: { @@ -273,7 +607,7 @@ describe('runUsage — error handling', () => { }); it('re-maps INSUFFICIENT_CREDITS (rate_limited with credits sub-case) to exit 12', async () => { - writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { deps } = makeCapture(); const creditError = { error: { diff --git a/src/commands/usage.ts b/src/commands/usage.ts index 1980d26..4342758 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -2,15 +2,13 @@ * `testsprite usage` — show the account's credit balance and plan/entitlement * info as a proactive pre-flight before a large `test run` fan-out. * - * Backend note: the `GET /me` endpoint does NOT currently return credit balance - * or plan info. This command calls `/me` for auth-identity fields and surfaces - * the `credits` / `subPlan` fields when and only when the backend supplies them - * (forward-compat / absent-safe). A dedicated backend endpoint is a required - * follow-up. - * - * BACKEND FOLLOW-UP REQUIRED: - * Add `credits`, `subPlan` to the `/me` response, or add a dedicated - * `GET /api/cli/v1/usage` endpoint returning `{ credits, subPlan, creditsPerRun }`. + * Backend note: `GET /me` now includes the `credits` / `subPlan` projection + * (this shipped; the CLI's old "requires a backend update" + * wording is stale and was removed). This command calls `/me` and surfaces + * the `credits` / `subPlan` / `creditsPerRun` fields when and only when the + * backend response includes them — kept absent-safe/forward-compat since not + * every account shape is guaranteed to populate all three (e.g. `creditsPerRun` + * has no server-side source at all today). */ import { Command } from 'commander'; @@ -23,28 +21,30 @@ import { import { loadConfig } from '../lib/config.js'; import { resolvePortalBase } from '../lib/facade.js'; import type { FetchImpl } from '../lib/http.js'; +import type { CliOrgBinding, CliOrgSummary } from '../lib/org-render.js'; +import { formatOrgBinding, formatOrgsSummary } from '../lib/org-render.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; /** * Usage/balance response from `/me` (when the backend supplies it) or a future * `/usage` endpoint. * - * All fields except `userId`/`keyId`/`env` are forward-compat: the backend - * does not return them today. They are rendered only when present. + * `credits` / `subPlan` now ship on `/me` (live). Still + * kept optional/absent-safe: `userId`/`keyId`/`env` are the only fields every + * backend and account shape is guaranteed to populate. */ export interface UsageResponse { userId: string; keyId: string; env: 'development' | 'staging' | 'production'; /** - * Remaining credit balance. Present only when the backend /me (or /usage) - * includes the User.credits projection. BACKEND FOLLOW-UP: me.controller.ts. + * Remaining credit balance. Present when the backend /me (or /usage) + * includes the User.credits projection (live). */ credits?: number; /** - * Subscription plan name (e.g. "Free", "Standard", "Pro"). Present only when - * the backend /me (or /usage) includes the User.subPlan projection. - * BACKEND FOLLOW-UP: me.controller.ts. + * Subscription plan name (e.g. "Free", "Standard", "Pro"). Present when + * the backend /me (or /usage) includes the User.subPlan projection (live). */ subPlan?: string; /** @@ -52,6 +52,46 @@ export interface UsageResponse { * backend supplies it. */ creditsPerRun?: number; + /** + * The caller's organization wallet — the billing subject on org-based + * accounts. Rendered only together with `v3Enabled: true` (see + * `renderUsage`): the org wallet supersedes the legacy `credits`/`subPlan` + * pair only for callers whose commands actually bill it. Absent-safe like + * every other optional field. + */ + activeOrg?: ActiveOrg; + /** + * Authoritative per-caller routing bit: true when this caller's commands + * run (and bill) on the V3 platform. Always present on current backends; + * absent on older ones. + */ + v3Enabled?: boolean; + /** + * Every organization the underlying user belongs to (account-wide + * membership list, personal org included). Absent-safe: omitted on a + * server-side lookup failure or an older backend. + */ + organizations?: CliOrgSummary[]; + /** + * The calling key's own org binding. Present only when the request + * authenticated with a Postgres-backed membership key (`sk-member-…`). + */ + org?: CliOrgBinding; +} + +/** Slim org-wallet view shipped on `/me` (see the backend `Me` schema). */ +export interface ActiveOrg { + id: string; + name: string; + /** Org plan (`Free` | `Starter` | `Standard`). */ + plan: string; + /** Caller's role in the org (`owner` | `admin` | `member`). */ + role: string; + /** Spendable balance: the caller's member bucket + the org's shared top-up pool. */ + remaining: number; + /** Monthly per-seat credit allowance for the org's plan. */ + includedCredits: number; + seats: number; } export interface UsageDeps { @@ -72,12 +112,23 @@ export const DRY_RUN_USAGE_SAMPLE: UsageResponse = { credits: 42, subPlan: 'Standard', creditsPerRun: 2, + v3Enabled: true, + activeOrg: { + id: '22222222-2222-4222-8222-222222222222', + name: 'Dry Run Workspace', + plan: 'Standard', + role: 'owner', + remaining: 1650, + includedCredits: 1600, + seats: 1, + }, }; /** * Run the `usage` command. Calls `GET /me`, surfaces identity + any * credits/plan fields the backend supplies. Absent fields are silently - * omitted (forward-compat until the backend adds the projection). + * omitted (forward-compat in case a given account/backend version doesn't + * populate them). */ export async function runUsage(opts: CommonOptions, deps: UsageDeps = {}): Promise { const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); @@ -85,7 +136,7 @@ export async function runUsage(opts: CommonOptions, deps: UsageDeps = {}): Promi if (opts.dryRun) { emitDryRunBanner(stderr); - stderr('[note] credit balance requires a backend update — showing dry-run sample values'); + stderr('[note] --dry-run showing canned sample values, not your real balance'); out.print(DRY_RUN_USAGE_SAMPLE, data => renderUsage(data as UsageResponse)); return DRY_RUN_USAGE_SAMPLE; } @@ -114,25 +165,50 @@ export async function runUsage(opts: CommonOptions, deps: UsageDeps = {}): Promi ? `${portalBase}/dashboard/settings/billing` : 'the portal Billing page (/dashboard/settings/billing)'; - // /me is the only available source of credits/plan today. - // When the backend adds credits/subPlan to MeResponse (or adds /usage), - // this single get call is sufficient — no code change needed in the CLI. + // /me is the only available source of credits/plan today. If the backend + // adds a dedicated /usage endpoint later, this single get call is where + // it would be swapped in — no other code change needed in the CLI. const me = await client.get('/me'); out.print(me, data => renderUsage(data as UsageResponse, portalBase)); - // In text mode, emit a backend-gap note when credits are missing so the - // user knows why the balance isn't shown (instead of assuming zero or error). - if (opts.output === 'text' && me.credits === undefined) { + // In text mode, emit a note when NO balance was shown at all — neither the + // legacy per-user `credits` nor an org wallet. A V3-routed org account can + // legitimately have no DDB `credits` field (org-native members never get a + // legacy user row), and its balance already rendered in the organization + // block — the note would contradict the output right above it. + // + // "Is this key org-bound" and "do I have a balance number to show" are two + // different questions and must not be conflated. `me.org` is the key's own + // binding — always present, no I/O, populated whenever the request + // authenticated with a membership key (`sk-member-…`). `activeOrg` is the + // *enriched* org balance: the backend does a best-effort Postgres read for + // it and swallows the exception on failure, so `org` can be present while + // `activeOrg` (and, for a V3-native user with no legacy DynamoDB row, + // `credits` too) is absent. Deciding org-boundedness from `activeOrg` + // (i.e. `orgWalletShown`) alone means that degraded state falls through to + // this personal-billing note — pointing an org-key operator at the wrong + // wallet in exactly the situation this whole surface exists to prevent. + const isOrgBound = me.org !== undefined; + const orgWalletShown = me.v3Enabled === true && me.activeOrg !== undefined; + if (opts.output === 'text' && me.credits === undefined && !orgWalletShown) { stderr( - '[note] credit balance not available — backend does not yet expose credits on /me.' + - ` Check ${billingUrl} for your current balance.`, + isOrgBound + ? "[note] this key is organization-bound, but the org balance could not be loaded right now. Retry `testsprite usage`, or check the Portal's organization billing settings (ask an org admin if you don't have access)." + : `[note] credit balance not returned for this account. Check ${billingUrl} for your current balance.`, ); } return me; } +/** + * Org-wallet low-balance threshold: the cost of a generation action — + * the priciest common single action on org billing. Below this, the next + * AI-assisted operation may fail; cheaper actions can still succeed. + */ +const LOW_ORG_BALANCE_CREDITS = 2; + function renderUsage(u: UsageResponse, portalBase?: string): string { const lines: string[] = []; @@ -140,10 +216,68 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { lines.push(`userId: ${u.userId}`); lines.push(`keyId: ${u.keyId}`); lines.push(`env: ${u.env}`); + // Org attribution — rendered only when the backend supplies it. + const orgsSummary = formatOrgsSummary(u.organizations); + if (orgsSummary) lines.push(`orgs: ${orgsSummary}`); + const orgBinding = formatOrgBinding(u.org); + if (orgBinding) lines.push(`org binding: ${orgBinding}`); + + // Whether the CALLING KEY is org-bound — read from `u.org` (the key's own + // membership binding: always present, no I/O, populated whenever the + // request authenticated with a `sk-member-…` key), NOT from `activeOrg`. + // `activeOrg` is the *enriched* org balance: the backend does a best-effort + // Postgres read for it and swallows the exception on failure, so a caller + // can be genuinely org-bound (`org` present) while `activeOrg` — and, for a + // V3-native user with no legacy DynamoDB row, `credits` too — is absent. + // "Is this key org-bound" and "do I have a balance number to show" are two + // different questions; conflating them (deciding org-boundedness from + // `orgWallet`/`activeOrg` alone) sends a degraded-enrichment org caller + // down the personal-wallet branches below. + const isOrgBound = u.org !== undefined; + + // Org wallet block — the billing subject on org-based accounts. Rendered + // only when the caller is actually V3-routed: `v3Enabled` is the + // authoritative routing bit, so wallet selection never rests on field + // presence alone. A V2-routed caller keeps the legacy block below (their + // billable commands still charge the legacy wallet); older backends send + // neither field and degrade the same way. + // `?? undefined` also normalizes a hypothetical explicit `null` from the + // wire so the block below can't dereference it. + const orgWallet = u.v3Enabled === true ? (u.activeOrg ?? undefined) : undefined; + if (orgWallet !== undefined) { + const org = orgWallet; + lines.push(''); + lines.push('--- organization ---'); + lines.push(`org: ${org.name} (${org.role})`); + lines.push(`plan: ${org.plan}`); + // Labeled `balance:` (not `credits:`) — `--output json` exposes the + // legacy per-user number under `.credits`, and giving the org wallet the + // same label in text mode would make one word mean two different values. + lines.push(`balance: ${org.remaining} remaining (${org.includedCredits}/mo per seat)`); + lines.push(`seats: ${org.seats}`); + // No "~N runs" estimate here: org billing prices actions individually and + // the API does not expose a per-run rate for the org wallet — an estimate + // computed from the legacy frontend rate would be wrong. + } else if (isOrgBound) { + // Org-bound key, but the enrichment that would have populated `activeOrg` + // degraded (best-effort Postgres read failed server-side, or `v3Enabled` + // itself couldn't be resolved). Say so honestly — never fall through to + // the legacy per-user blocks below (this key's commands bill the org + // wallet, not the personal one, regardless of whether we could load its + // number just now), and never fabricate an org-scoped URL. + lines.push(''); + lines.push('--- organization ---'); + lines.push( + "balance: could not be loaded right now. Retry `testsprite usage`, or check the Portal's organization billing settings (ask an org admin if you don't have access).", + ); + } - // Balance block — shown only when the backend supplies it + // Legacy balance block — shown only when the backend supplies it, no org + // wallet superseded it (older backends / V2-routed accounts), AND the key + // isn't org-bound (an org-bound key's commands never charge these numbers, + // even if a legacy row happens to still carry them). const hasBalanceData = u.credits !== undefined || u.subPlan !== undefined; - if (hasBalanceData) { + if (orgWallet === undefined && !isOrgBound && hasBalanceData) { lines.push(''); lines.push('--- credits & plan ---'); if (u.subPlan !== undefined) { @@ -154,7 +288,7 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { } if (u.creditsPerRun !== undefined) { lines.push(`cost per frontend run: ${u.creditsPerRun} credit(s)`); - // Backend runs DO consume credits (confirmed by design 2026-06-30 / DEV-289). + // Backend runs DO consume credits (confirmed by design 2026-06-30). // The API exposes no backend-specific per-run cost field, and it differs from // the frontend rate, so state that it bills without asserting a possibly-wrong // number — check your balance before/after, or see the billing page. @@ -170,16 +304,39 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { } } - // Actionable upgrade line for Free or low-balance keys + // Actionable upgrade line for Free or low-balance keys. Prefer the org + // wallet's plan/balance when present. `!isOrgBound` guards the personal + // branch of each: an org-bound key with degraded enrichment has no + // `orgWallet` to compute from, but must not fall back to reading `u.credits` + // / `u.subPlan` either (a legacy row that happens to coexist with an org + // binding is not what this key's commands actually bill). const isLowBalance = - u.credits !== undefined && u.creditsPerRun !== undefined && u.credits < u.creditsPerRun; - const isFree = u.subPlan?.toLowerCase() === 'free'; + orgWallet !== undefined + ? orgWallet.remaining < LOW_ORG_BALANCE_CREDITS + : !isOrgBound && + u.credits !== undefined && + u.creditsPerRun !== undefined && + u.credits < u.creditsPerRun; + const isFree = + orgWallet !== undefined + ? orgWallet.plan.toLowerCase() === 'free' + : !isOrgBound && u.subPlan?.toLowerCase() === 'free'; if (isLowBalance) { lines.push(''); + // The org wallet is billed under the ORGANIZATION's own settings, not the + // personal `/dashboard/settings/billing` page (that page manages the + // legacy per-user DDB balance, a different column entirely) — so the + // org branch deliberately does not point at that URL. No org-scoped + // settings URL is fabricated here either: the CLI has no confirmed route + // for one, so "ask an org admin" is the honest next step. lines.push( - 'warning: credit balance is below the per-run cost. Top up at:' + - ` ${portalBase !== undefined ? `${portalBase}/dashboard/settings/billing` : 'the portal Billing page (/dashboard/settings/billing)'}`, + orgWallet !== undefined + ? // Org billing prices actions individually, and cheaper actions (e.g. + // a 1-credit backend run) may still succeed below the threshold — + // so this is "low", not "cannot run". + `warning: organization balance is low (under the ${LOW_ORG_BALANCE_CREDITS}-credit cost of a generation action). Top up in the Portal's organization billing settings (ask an org admin if you don't have access).` + : `warning: credit balance is below the per-run cost. Top up at: ${portalBase !== undefined ? `${portalBase}/dashboard/settings/billing` : 'the portal Billing page (/dashboard/settings/billing)'}`, ); } else if (isFree) { lines.push(''); @@ -210,8 +367,10 @@ export function createUsageCommand(deps: UsageDeps = {}): Command { ' 0 success (or --dry-run)\n' + ' 3 auth error — run `testsprite setup` to configure credentials\n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + - '\nNote: credit balance requires a backend update to /me. Until shipped,\n' + - " check your portal's Billing page (/dashboard/settings/billing) for your balance.", + "\nNote: if credit balance isn't shown for your account, check your portal's\n" + + ' Billing page (/dashboard/settings/billing) for a personal key, or your\n' + + ' organization billing settings (ask an org admin) if this key is\n' + + ' organization-bound.', ) .action(async (_cmdOpts, command: Command) => { await runUsage(resolveCommonOptions(command), deps); diff --git a/src/index.ts b/src/index.ts index 31d3399..bc5bd9a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,19 +6,33 @@ import { createAuthCommand } from './commands/auth.js'; import { createCompletionCommand, type CompletionSpec } from './commands/completion.js'; import { createDoctorCommand } from './commands/doctor.js'; import { + addSetupOptions, createDeprecatedInitCommand, createSetupCommand, runConfigureViaSetup, + type SetupCmdOpts, } from './commands/init.js'; import { createProjectCommand } from './commands/project.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; +import { TARGETS, type AgentTarget } from './lib/agent-targets.js'; import { ApiError, CLIError, InterruptError, RequestTimeoutError } from './lib/errors.js'; import { installBrokenPipeGuard, installSignalHandlers } from './lib/interrupt.js'; import { Output, isOutputMode } from './lib/output.js'; import { maybeInstallProxyAgent } from './lib/proxy.js'; -import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; -import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; +import { + renderAmbiguousOrgCandidates, + renderCommanderError, + rephraseUnknownOption, +} from './lib/render-error.js'; +import { isPlanTemplateInvocation, maybeEmitSkillNudge } from './lib/skill-nudge.js'; +import { + classifyCliError, + isTelemetryOptedOut, + recordOutcome, + resolveTelemetryAuth, + type ResolvedTelemetryAuth, +} from './lib/telemetry.js'; import { maybeNotifyUpdate } from './lib/update-check.js'; import { VERSION } from './version.js'; import { shouldRejectNodeVersion } from './version-guard.js'; @@ -71,21 +85,23 @@ program.addCommand(createDeprecatedInitCommand({}), { hidden: true }); // `auth configure` is a hidden, deprecated alias that runs FULL `setup` // (configure + skill install), so an agent reaching for the old command still // ends up with the skill. `setup` remains the ONLY path that writes credentials. +// Attach the SAME flag set `setup` has (previously only +// `--from-env` was wired up, so README's "runs the full setup" claim didn't +// hold — `--yes`/`--agent`/`--api-key`/`--force`/`--dir`/`--no-agent` were all +// rejected as `unknown option`). const authCommand = createAuthCommand(); -authCommand - .command('configure', { hidden: true }) - .option( - '--from-env', - 'Read TESTSPRITE_API_KEY (and optionally TESTSPRITE_API_URL) from the environment instead of prompting', - false, - ) - .action(async (cmdOpts: { fromEnv?: boolean }, command: Command) => { - process.stderr.write( - '[deprecated] `testsprite auth configure` now runs full setup (configure + skill install) — ' + - 'use `testsprite setup` (add --no-agent to skip the skill).\n', - ); - await runConfigureViaSetup(command, {}, Boolean(cmdOpts.fromEnv)); - }); +const authConfigureValidTargets = Object.keys(TARGETS) as AgentTarget[]; +addSetupOptions( + authCommand.command('configure', { hidden: true }), + authConfigureValidTargets, + 'claude', +).action(async (cmdOpts: SetupCmdOpts, command: Command) => { + process.stderr.write( + '[deprecated] `testsprite auth configure` now runs full setup (configure + skill install) — ' + + 'use `testsprite setup` (add --no-agent to skip the skill).\n', + ); + await runConfigureViaSetup(command, {}, cmdOpts); +}); program.addCommand(authCommand); program.addCommand(createProjectCommand({})); @@ -122,6 +138,20 @@ function buildCompletionSpec(): CompletionSpec { // command throws CommanderError rather than calling process.exit directly. let pendingCommanderErrorMsg: string | null = null; +// Leaf command path that actually reached an action (set by the preAction hook +// below). Empty when no action ran (bare `--help`, parse error) — telemetry +// skips those. Read by the success + error telemetry emits around parseAsync. +let ranCommandPath = ''; + +// Whether the command that ran should emit telemetry. `completion` (its stdout +// is eval'd by shells) and `--plan-template` (pure-local) are kept off the +// beacon, same as the update notice. Set by the preAction hook. +let telemetryEmit = false; + +// Auth resolved in the preAction hook, before the command's action runs, so +// `auth remove` (which deletes the profile) is still reported on the key it used. +let telemetryAuth: ResolvedTelemetryAuth | undefined; + // Propagate exitOverride AND the buffered outputError config to every // subcommand in the tree. Commander's addCommand() does NOT inherit either // from the parent, so commands built externally (createTestCommand, etc.) and @@ -156,6 +186,22 @@ function commandPathOf(cmd: Command): string { return names.join(' '); } +/** Global flags the telemetry emit needs (config resolution + context). */ +function telemetryGlobals(): { + profile?: string; + endpointUrl?: string; + output?: string; + dryRun?: boolean; +} { + const g = program.opts<{ + profile?: string; + endpointUrl?: string; + output?: string; + dryRun?: boolean; + }>(); + return { profile: g.profile, endpointUrl: g.endpointUrl, output: g.output, dryRun: g.dryRun }; +} + // Best-effort onboarding nudge (see lib/skill-nudge.ts): when a configured // caller drives a verify-loop command in a project with no installed skill, // point it at `testsprite setup`. A preAction hook runs before every leaf @@ -165,24 +211,53 @@ program.hook('preAction', (_thisCommand, actionCommand) => { const globals = actionCommand.optsWithGlobals() as { output?: string; profile?: string; + endpointUrl?: string; dryRun?: boolean; + planTemplate?: boolean; }; const commandPath = commandPathOf(actionCommand); - maybeEmitSkillNudge({ - commandPath, - output: isOutputMode(globals.output) ? globals.output : 'text', - dryRun: globals.dryRun ?? false, - profile: globals.profile ?? 'default', - cwd: process.cwd(), - env: process.env, - }); + // Record which leaf command ran, for the telemetry emit around parseAsync. + ranCommandPath = commandPath; + // See `isPlanTemplateInvocation` for why this one case is + // filtered here rather than in skill-nudge.ts / update-check.ts. + const isPlanTemplate = isPlanTemplateInvocation(commandPath, globals.planTemplate); + + // Telemetry gate, same reasons as the update notice below: skip `completion` + // (its stdout is eval'd by shells) and `--plan-template` (pure-local). Resolve + // auth now, before the command's action runs, so `auth remove` still reports + // on the key it used; skip the read for opt-out / dry-run. + telemetryEmit = commandPath !== 'completion' && !isPlanTemplate; + if (telemetryEmit && !isTelemetryOptedOut(process.env) && globals.dryRun !== true) { + telemetryAuth = resolveTelemetryAuth({ + profile: globals.profile, + endpointUrl: globals.endpointUrl, + }); + } + if (!isPlanTemplate) { + maybeEmitSkillNudge({ + commandPath, + output: isOutputMode(globals.output) ? globals.output : 'text', + dryRun: globals.dryRun ?? false, + profile: globals.profile ?? 'default', + cwd: process.cwd(), + env: process.env, + }); + } // Best-effort update notice (see lib/update-check.ts): self-gates on the // opt-out env, CI, TTY, and a 24h cache; the wiring adds the flag-level // gates the lib cannot see. Skipped for `completion` (its stdout is eval'd - // by shells), under --output json, and under --dry-run. Deliberately not - // awaited: an advisory must never delay the real command. - if (globals.output !== 'json' && globals.dryRun !== true && commandPath !== 'completion') { + // by shells), under --output json, under --dry-run, and for + // `--plan-template` (pure-local — this check hits the npm registry over + // the network and writes ~/.testsprite/update-check.json, both of which + // contradict "no network" for this flag). Deliberately not awaited: an + // advisory must never delay the real command. + if ( + globals.output !== 'json' && + globals.dryRun !== true && + commandPath !== 'completion' && + !isPlanTemplate + ) { void maybeNotifyUpdate(); } }); @@ -202,9 +277,46 @@ installBrokenPipeGuard(); // ignores them by default). No-op when no proxy variable is set. maybeInstallProxyAgent(); +const telemetryStartedAt = Date.now(); try { await program.parseAsync(process.argv); + // Flush a success outcome event before the process exits 0. Bounded + + // best-effort (see lib/telemetry.ts); skips when no command ran / no key + // configured / opted out / --dry-run. + if (telemetryEmit) { + await recordOutcome( + { + command: ranCommandPath, + outcome: 'success', + exitCode: 0, + durationMs: Date.now() - telemetryStartedAt, + ...telemetryGlobals(), + }, + { resolvedAuth: telemetryAuth }, + ); + } } catch (err) { + const telemetryOutcome = classifyCliError(err); + // Flush the outcome AFTER the error is rendered to stderr below (via each + // branch's `flushThenExit`), so a slow backend never delays the user's error. + // The classification mirrors the exit-code mapping the branches apply. + const flushThenExit = async (code: number): Promise => { + if (telemetryEmit) { + await recordOutcome( + { + command: ranCommandPath, + outcome: telemetryOutcome.outcome, + exitCode: telemetryOutcome.exitCode, + errorCode: telemetryOutcome.errorCode, + durationMs: Date.now() - telemetryStartedAt, + ...telemetryGlobals(), + }, + { resolvedAuth: telemetryAuth }, + ); + } + process.exit(code); + }; + const rawMode = program.opts<{ output?: string }>().output; const mode = isOutputMode(rawMode) ? rawMode : 'text'; if (err instanceof ApiError) { @@ -245,8 +357,16 @@ try { process.stderr.write(` your version: ${your}, minimum supported: ${min}\n`); } } + // AMBIGUOUS_ORG: the same testId resolved inside more than one of the + // caller's organizations. List each colliding candidate plus a hint to + // disambiguate, mirroring the AUTH_FORBIDDEN required/granted pattern above. + if (err.code === 'AMBIGUOUS_ORG') { + for (const line of renderAmbiguousOrgCandidates(err.getDetail('candidates'))) { + process.stderr.write(`${line}\n`); + } + } } - process.exit(err.exitCode); + await flushThenExit(err.exitCode); } const output = new Output(mode); if (err instanceof InterruptError) { @@ -271,7 +391,7 @@ try { } else { process.stderr.write(`Error: ${err.message}\n`); } - process.exit(err.exitCode); + await flushThenExit(err.exitCode); } if (err instanceof RequestTimeoutError) { // Structured rendering for per-request timeouts: JSON mode emits a @@ -292,7 +412,7 @@ try { } else { process.stderr.write(`Error: ${err.message}\n`); } - process.exit(err.exitCode); + await flushThenExit(err.exitCode); } if (err instanceof CommanderError) { // Map exit codes per the CLI taxonomy: @@ -313,7 +433,7 @@ try { err.code === 'commander.help' || err.code === 'commander.version' ) { - process.exit(0); + await flushThenExit(0); } // For parse errors, write the buffered message in the correct format. // rawMode from program.opts() is reliable when --output was parsed before @@ -332,12 +452,12 @@ try { process.stderr.write( renderCommanderError(pendingCommanderErrorMsg, err.message, commanderMode), ); - process.exit(5); + await flushThenExit(5); } if (err instanceof CLIError) { output.error(err.message); - process.exit(err.exitCode); + await flushThenExit(err.exitCode); } output.error(err instanceof Error ? err.message : String(err)); - process.exit(1); + await flushThenExit(1); } diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 7e6f94d..f973403 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -3,14 +3,7 @@ import { readFileSync } from 'node:fs'; import { VERSION } from '../version.js'; export type AgentTarget = - | 'claude' - | 'cursor' - | 'cline' - | 'antigravity' - | 'codex' - | 'kiro' - | 'windsurf' - | 'copilot'; + 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex' | 'kiro' | 'windsurf' | 'copilot'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -59,9 +52,7 @@ export interface TargetSpec { * - 'none': skill is not represented in AGENTS.md at all (reserved). */ export type CodexContribution = - | { kind: 'full'; file: string } - | { kind: 'line'; text: string } - | { kind: 'none' }; + { kind: 'full'; file: string } | { kind: 'line'; text: string } | { kind: 'none' }; export interface SkillSpec { /** Skill name — appears in own-file frontmatter and the landing path. */ diff --git a/src/lib/api-key-prefix-leak-guard.test.ts b/src/lib/api-key-prefix-leak-guard.test.ts new file mode 100644 index 0000000..87ab2a2 --- /dev/null +++ b/src/lib/api-key-prefix-leak-guard.test.ts @@ -0,0 +1,84 @@ +import { readFileSync, existsSync } from 'node:fs'; +import { describe, it, expect } from 'vitest'; +import { API_KEY_PREFIXES } from './client-factory.js'; + +/** + * Drift guard between "what counts as a TestSprite API key" and "what our + * release-time leak scans can see". + * + * The 2026-08-01 rename to `sk-member-` made every newly minted key invisible + * to all three detectors at once — `.gitleaks.toml` had no provider rule at + * all, and both LEAK_RE greps enumerated `sk-user-` only. A leak of the + * CURRENT credential would have passed the public-export gate. After an + * incident that put 452 live keys in 499 public repositories, that is the + * failure that only surfaces the next time. + * + * So: every prefix the CLI accepts must be a prefix the scans detect, checked + * mechanically rather than remembered. + */ + +/** + * Realistic tokens for `prefix` — 43 base64url chars, what both mint paths + * emit. `tsp_` is a NAMESPACE, not a token prefix: the format gate accepts any + * `tsp_`-prefixed value, and `tsp_sa_` is reserved for phase-2 service-account + * keys. Probing only `tsp_u_` would let a scan that pins `u_` pass while the + * CLI happily accepts a `tsp_sa_` credential no detector can see — the exact + * failure this file exists to prevent, one level down. Add a kind here in the + * same change that starts minting it. + */ +const TSP_KINDS = ['u', 'sa'] as const; +const samples = (prefix: string): string[] => + prefix === 'tsp_' + ? TSP_KINDS.map(k => `${prefix}${k}_${'A'.repeat(43)}`) + : [`${prefix}${'A'.repeat(43)}`]; + +describe('API key prefixes are covered by the release leak scans', () => { + it('every accepted prefix is matched by a .gitleaks.toml rule', () => { + const toml = readFileSync('.gitleaks.toml', 'utf8'); + // Crude but sufficient: pull every `regex = '''…'''` out of the rule + // blocks. A structured TOML parse would need a new dependency for one + // assertion, and the failure mode we care about (a prefix nobody added) + // shows up identically either way. + const regexes: string[] = [...toml.matchAll(/^regex\s*=\s*'''(.*)'''\s*$/gm)].map( + m => m[1] as string, + ); + expect(regexes.length).toBeGreaterThan(0); + for (const prefix of API_KEY_PREFIXES) { + for (const token of samples(prefix)) { + const covered = regexes.some(r => new RegExp(r).test(token)); + expect(covered, `no gitleaks rule detects "${token.slice(0, 12)}…"`).toBe(true); + } + } + }); + + it('every accepted prefix is DETECTED by the copybara leak-safety LEAK_RE', () => { + // `scripts/make-public-snapshot.sh` carries the same patterns but is DROPped + // from the public snapshot, so it cannot be asserted from a shipped test. + // `copybara/leak-safety-harness.sh` ships, and the two are kept in sync by + // hand — if you edit one, edit both. + // + // This used to assert only that the `sk-` families were *enumerated*, and + // skipped `tsp_` on the reasoning that "gitleaks covers it". A post-merge + // audit showed that reasoning is wrong where it matters: the snapshot + // script treats gitleaks as optional-skip-not-fail, so on a local + // break-glass run LEAK_RE is the ONLY detector — and it never matched a + // `tsp_` token. So the assertion is now behavioural: build a realistic + // token for every accepted prefix and require the actual pattern to match + // it. Enumeration was a proxy; detection is the property. + const path = 'copybara/leak-safety-harness.sh'; + if (!existsSync(path)) return; + const sh = readFileSync(path, 'utf8'); + const patterns = [...sh.matchAll(/^[A-Z_]*LEAK_RE='([^']+)'/gm)].map(m => m[1] as string); + expect(patterns.length).toBeGreaterThan(0); + for (const prefix of API_KEY_PREFIXES) { + for (const token of samples(prefix)) { + for (const pattern of patterns) { + expect( + new RegExp(pattern).test(token), + `a LEAK_RE does not detect "${token.slice(0, 14)}…"`, + ).toBe(true); + } + } + } + }); +}); diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index d954381..34aaf40 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -20,6 +20,7 @@ import { buildMeta, isBundleOwnedEntry, pickCodeExtension, + pickVideoExtension, resolveBundleDir, STREAM_URL_MAX_RETRIES, streamUrlToFile, @@ -539,6 +540,34 @@ describe('pickCodeExtension', () => { }); }); +describe('pickVideoExtension', () => { + it('matches the actual recording container instead of hardcoding mp4', () => { + expect(pickVideoExtension('https://s3.example.com/run_abc.webm')).toBe('webm'); + expect(pickVideoExtension('https://s3.example.com/run_abc.mp4')).toBe('mp4'); + }); + + it('strips presigned-URL query strings before reading the extension', () => { + expect( + pickVideoExtension( + 'https://s3.example.com/run_abc.webm?X-Amz-Signature=abc&X-Amz-Expires=60', + ), + ).toBe('webm'); + }); + + it('is case-insensitive', () => { + expect(pickVideoExtension('https://s3.example.com/run_abc.WEBM')).toBe('webm'); + }); + + it('falls back to mp4 for an unrecognized or missing extension', () => { + expect(pickVideoExtension('https://s3.example.com/run_abc.bin')).toBe('mp4'); + expect(pickVideoExtension('https://s3.example.com/run_abc')).toBe('mp4'); + }); + + it('falls back to mp4 on a malformed URL rather than throwing', () => { + expect(pickVideoExtension('not a url')).toBe('mp4'); + }); +}); + describe('stepFilenamePrefix', () => { it('zero-pads to 2 digits for index < 100', () => { expect(stepFilenamePrefix(1)).toBe('01'); @@ -771,6 +800,14 @@ describe('isBundleOwnedEntry', () => { expect(isBundleOwnedEntry('code.py')).toBe(true); }); + it('owns video. only for the writable-extension allowlist', () => { + expect(isBundleOwnedEntry('video.mp4')).toBe(true); + expect(isBundleOwnedEntry('video.webm')).toBe(true); + expect(isBundleOwnedEntry('video.mov')).toBe(true); + expect(isBundleOwnedEntry('video.mkv')).toBe(true); + expect(isBundleOwnedEntry('video.avi')).toBe(true); + }); + it('does not own foreign entries', () => { expect(isBundleOwnedEntry('notes.txt')).toBe(false); expect(isBundleOwnedEntry('src')).toBe(false); @@ -778,6 +815,13 @@ describe('isBundleOwnedEntry', () => { expect(isBundleOwnedEntry('code.tar.gz')).toBe(false); expect(isBundleOwnedEntry('mycode.ts')).toBe(false); expect(isBundleOwnedEntry('code.')).toBe(false); + expect(isBundleOwnedEntry('video.tar.gz')).toBe(false); + expect(isBundleOwnedEntry('myvideo.mp4')).toBe(false); + expect(isBundleOwnedEntry('video.')).toBe(false); + // A user's own file in a pre-existing --out dir must never be swept + // just because it starts with `video.`. + expect(isBundleOwnedEntry('video.txt')).toBe(false); + expect(isBundleOwnedEntry('video.MP4')).toBe(false); }); }); @@ -925,6 +969,25 @@ describe('step artifact path validation', () => { expect(existsSync(join(dir, 'video.mp4'))).toBe(false); }); + it('sweeps a stale video file with a different extension than the new bundle writes', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'video.mp4'), 'stale-mp4-bytes', 'utf8'); + + const webmCtx: CliFailureContext = { + ...stepCtx(3), + result: { ...stepCtx(3).result, videoUrl: 'https://video.example.com/run_abc.webm' }, + }; + const fetchImpl = (async () => + new Response('fake-webm-bytes', { status: 200 })) as unknown as typeof globalThis.fetch; + + const res = await writeBundle(webmCtx, { dir, failedOnly: false, fetchImpl }); + + expect(res.files).toContain('video.webm'); + expect(existsSync(join(dir, 'video.webm'))).toBe(true); + expect(readFileSync(join(dir, 'video.webm'), 'utf8')).toBe('fake-webm-bytes'); + expect(existsSync(join(dir, 'video.mp4'))).toBe(false); + }); + it('sweeps a stale code file with a different extension than the new bundle writes', async () => { const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); writeFileSync(join(dir, 'code.py'), '# stale python code\n', 'utf8'); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index 0685fd7..b02e164 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -364,6 +364,28 @@ export function pickCodeExtension(language: string, framework: string): string { return 'py'; } +/** Recording container extensions the bundle writer will trust from `videoUrl`. */ +const KNOWN_VIDEO_EXTENSIONS = new Set(['mp4', 'webm', 'mov', 'mkv', 'avi']); + +/** + * Pick the on-disk extension for `/video.` from the actual + * `result.videoUrl`, instead of hardcoding `mp4` (the + * recording is sometimes a `.webm` container; saving it under a `video.mp4` + * name misleads strict-extension consumers). Falls back to `mp4` when the + * URL is malformed or its extension isn't a recognized container, so a + * response-controlled field can never smuggle an arbitrary suffix onto disk. + */ +export function pickVideoExtension(url: string): string { + try { + const match = /\.([A-Za-z0-9]+)$/.exec(new URL(url).pathname); + const ext = match?.[1]?.toLowerCase(); + if (ext !== undefined && KNOWN_VIDEO_EXTENSIONS.has(ext)) return ext; + } catch { + // Malformed URL — fall through to the default below. + } + return 'mp4'; +} + /** * Step filename per §7.2 — 1-based index, zero-padded to two digits * for indices ≤ 99, three digits for ≥ 100. `${stepIndex}-snapshot.html` @@ -421,8 +443,9 @@ export function buildMeta(ctx: CliFailureContext, fetchedAt: Date = new Date()): * 2. `applyFailedOnly` — narrow before download (saves bytes). * 3. `mkdir /.tmp/` — fresh; clean any stale temp. * 4. `writeFile result.json / failure.json / code.` — local data. - * 5. `fetch + stream` for `video.mp4` (when set) and per-step - * snapshot/screenshot/evidence-json files. + * 5. `fetch + stream` for `video.` (when set; extension matches the + * actual recording container) and per-step snapshot/screenshot/ + * evidence-json files. * 6. `writeFile meta.json` LAST — its presence means "bundle complete". * 7. `rename .tmp/ -> ` for every file. Last rename * makes meta.json visible. @@ -489,10 +512,12 @@ export async function writeBundle( // Optional video. Wire field is `result.videoUrl` (not in `failure`), // and the CLI surfaces it as a top-level on-disk artifact for agent // ergonomics — the agent doesn't have to know which sub-object held - // it on the wire. + // it on the wire. Extension matches the actual recording container + // (some runs ship `.webm`, not `.mp4`). if (filtered.result.videoUrl) { - await streamUrlToFile(filtered.result.videoUrl, join(tmpDir, 'video.mp4'), fetchImpl); - filesWritten.push('video.mp4'); + const videoFile = `video.${pickVideoExtension(filtered.result.videoUrl)}`; + await streamUrlToFile(filtered.result.videoUrl, join(tmpDir, videoFile), fetchImpl); + filesWritten.push(videoFile); } for (const step of filtered.steps) { @@ -540,17 +565,17 @@ async function freshTmpDir(dir: string): Promise { /** * Whether a top-level directory entry belongs to the bundle format — * i.e. something a prior `writeBundle` could have produced and this - * commit is therefore allowed to clean up. `code.` is matched by - * pattern (not the current run's extension) so a stale `code.py` is - * still swept when the new bundle writes `code.ts`. Everything else in - * the directory is the user's and must never be deleted (`--out` can - * point at a pre-existing, populated directory). + * commit is therefore allowed to clean up. `code.` and `video.` + * are matched by pattern (not the current run's extension) so a stale + * `code.py` or `video.webm` is still swept when the new bundle writes + * `code.ts` / `video.mp4` (or no video at all). Everything else in the + * directory is the user's and must never be deleted (`--out` can point at + * a pre-existing, populated directory). */ export function isBundleOwnedEntry(entry: string): boolean { if ( entry === 'result.json' || entry === 'failure.json' || - entry === 'video.mp4' || entry === 'meta.json' || entry === 'steps' || entry === '.tmp' || @@ -558,7 +583,14 @@ export function isBundleOwnedEntry(entry: string): boolean { ) { return true; } - return /^code\.[A-Za-z0-9]+$/.test(entry); + // Videos: only the extensions the bundle itself can write (see + // KNOWN_VIDEO_EXTENSIONS / pickVideoExtension). A broader `video.*` + // pattern would claim user files like `video.txt` sitting in a + // pre-existing `--out` directory and delete them in the sweep. + return ( + /^code\.[A-Za-z0-9]+$/.test(entry) || + (entry.startsWith('video.') && KNOWN_VIDEO_EXTENSIONS.has(entry.slice('video.'.length))) + ); } /** diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index d5135cc..f968a33 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -362,6 +362,59 @@ describe('makeHttpClient - API key validation', () => { expect(apiErr.nextAction).toContain('api-key'); }); + it.each([ + ['legacy key', 'sk-user-abcdef123'], + // Constructed, not literal token-shaped strings — keeps secret scanners quiet. + ['membership key', `sk-member-${'A'.repeat(43)}`], + ['pre-rename membership key', `tsp_u_${'A'.repeat(43)}`], + ])('accepts a well-formed TestSprite key (%s)', (_label, apiKey) => { + const fetchImpl = vi.fn().mockResolvedValue(new Response('{}', { status: 200 })); + expect(() => + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: apiKey } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ), + ).not.toThrow(); + }); + + it.each([ + ['no recognized prefix', 'not-a-valid-key-format!!'], + ['prefix only, empty body', 'sk-user-'], + ['membership prefix only, empty body', 'sk-member-'], + ['org prefix only, empty body', 'tsp_'], + ['wrong prefix', 'pk-user-abcdef'], + // `sk-` alone is not a family — it must not be treated as a valid prefix. + ['bare sk- prefix', 'sk-abcdef123'], + ])( + 'rejects a header-legal key with a bad TestSprite format (%s) before any network call', + (_label, apiKey) => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: apiKey } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }, + ); + it('treats a whitespace-only TESTSPRITE_API_KEY env var as unset (AUTH_REQUIRED)', () => { const fetchImpl = vi.fn(); let caught: unknown; diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index 96d9dc0..98d2f94 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -79,6 +79,21 @@ export interface ClientFactoryDeps { shutdownSignal?: AbortSignal; } +/** + * Accepted TestSprite API-key prefixes, in the order they are reported to the + * user. Three families exist today and all three authenticate: + * + * - `sk-user-…` legacy encrypted-envelope key — acts as the human + * - `sk-member-…` membership key — acts as ONE org membership of that human + * - `tsp_…` the pre-rename spelling of a membership key (`sk-member-…`); + * still issued to nobody, still valid for everyone holding one + * + * `sk-member-` is deliberately NOT shortened to `sk-` here: this list is only + * a fail-fast format check, but keeping the full prefixes means the message + * below can name exactly what the server will accept. + */ +export const API_KEY_PREFIXES = ['sk-user-', 'sk-member-', 'tsp_'] as const; + /** * The fake API key used in dry-run. Never sent — the dry-run fetch * impl ignores headers and returns a canned sample. Documented in the @@ -189,6 +204,32 @@ export function assertValidEndpointUrl(rawUrl: string): void { } } +/** + * Reject a key that isn't a TestSprite key, up front — so a malformed key + * fails fast instead of a slow live `fetch failed`. Checks only the prefix + + * non-empty body against {@link API_KEY_PREFIXES}, so it can never reject a + * real key of any issued family. + */ +export function assertValidApiKeyFormat(apiKey: string): void { + const matchesPrefix = (prefix: string): boolean => + apiKey.startsWith(prefix) && apiKey.length > prefix.length; + if (!API_KEY_PREFIXES.some(matchesPrefix)) { + const shown = API_KEY_PREFIXES.map(p => `\`${p}\``).join(', '); + throw localValidationError( + 'api-key', + `must be a TestSprite API key beginning with one of ${shown} (create one in the dashboard, then run \`testsprite setup\`)`, + undefined, + 'field', + ); + } +} + +/** Up-front key validation: TestSprite format + legal HTTP header value. */ +export function assertValidApiKey(apiKey: string): void { + assertValidApiKeyFormat(apiKey); + assertValidApiKeyHeaderValue(apiKey); +} + export function assertValidApiKeyHeaderValue(apiKey: string): void { const reason = 'must be a non-empty HTTP header value; paste the raw key without smart punctuation, emoji, or line breaks'; @@ -277,7 +318,7 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} // VALIDATION_ERROR rather than an opaque URL throw or a retried "fetch failed". assertValidEndpointUrl(config.apiUrl); if (!config.apiKey) throw ApiError.authRequired(); - assertValidApiKeyHeaderValue(config.apiKey); + assertValidApiKey(config.apiKey); return new HttpClient({ baseUrl: facadeBaseUrl(config.apiUrl), apiKey: config.apiKey, diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index 078c653..9f930da 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -316,8 +316,11 @@ describe('findSample', () => { break; } case 'createProject': - // P6 — POST /projects → CliProject shape. + // P6 — POST /projects → CliCreateProjectResponse shape. Both id + // field names present (the live field is + // `projectId`; `id` is kept for back-compat). expect(body).toMatchObject({ + projectId: expect.any(String), id: expect.any(String), type: expect.any(String), name: expect.any(String), @@ -326,8 +329,10 @@ describe('findSample', () => { }); break; case 'updateProject': - // P7 — PATCH /projects/{id} → CliUpdateProjectResponse shape. + // P7 — PATCH /projects/{id} → CliUpdateProjectResponse shape. Both + // id field names present. expect(body).toMatchObject({ + projectId: expect.any(String), id: expect.any(String), updatedFields: expect.any(Array), updatedAt: expect.any(String), diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index f6f5ec2..84c8e4e 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -18,6 +18,7 @@ */ import type { CliProject, + CliCreateProjectResponse, CliUpdateProjectResponse, CliDeleteProjectResponse, } from '../../commands/project.js'; @@ -110,6 +111,15 @@ const me: MeResponse = { scopes: ['read:projects', 'read:tests', 'write:tests', 'run:tests'], env: 'development', v3Enabled: true, + activeOrg: { + id: '22222222-2222-4222-8222-222222222222', + name: 'Dry Run Workspace', + plan: 'Standard', + role: 'owner', + remaining: 1650, + includedCredits: 1600, + seats: 1, + }, }; const projects: CliProject[] = [ @@ -389,16 +399,21 @@ const ENTRIES: DryRunSampleEntry[] = [ entry('getProject', 'GET', '/projects/{projectId}', projects[0]), // P6 — POST /projects (create project). The id uses a stable dry-run // sentinel so agents can see a coherent field shape without a real key. + // Both `projectId` (the live field) and `id` (legacy/ + // fallback) are shown — see `CliCreateProjectResponse`. entry('createProject', 'POST', '/projects', { + projectId: 'p_dryrun_create_2026', id: 'p_dryrun_create_2026', type: 'frontend', name: 'Dry-run project', createdFrom: 'cli', createdAt: '2026-05-16T00:00:00.000Z', updatedAt: '2026-05-16T00:00:00.000Z', - } satisfies CliProject), - // P7 — PATCH /projects/{id} (update project). + } satisfies CliCreateProjectResponse), + // P7 — PATCH /projects/{id} (update project). Both id + // field names shown — see `CliUpdateProjectResponse`. entry('updateProject', 'PATCH', '/projects/{projectId}', { + projectId: SAMPLE_PROJECT_ID, id: SAMPLE_PROJECT_ID, updatedFields: ['name'], updatedAt: '2026-05-16T00:00:00.000Z', diff --git a/src/lib/errors.test.ts b/src/lib/errors.test.ts index 5c5ba66..c957b05 100644 --- a/src/lib/errors.test.ts +++ b/src/lib/errors.test.ts @@ -61,6 +61,7 @@ describe('exitCodeFor', () => { ['INSUFFICIENT_CREDITS', 12], ['FEATURE_GATED', 13], ['CLIENT_TOO_OLD', 14], + ['AMBIGUOUS_ORG', 6], ['INTERNAL', 1], ] as const)('%s → exit %d', (code, expected) => { expect(exitCodeFor(code)).toBe(expected); @@ -137,6 +138,7 @@ describe('ApiError.fromEnvelope status fallback', () => { it.each([ [400, 'VALIDATION_ERROR' as const], [401, 'AUTH_INVALID' as const], + [402, 'INSUFFICIENT_CREDITS' as const], [403, 'AUTH_FORBIDDEN' as const], [404, 'NOT_FOUND' as const], [409, 'CONFLICT' as const], @@ -257,6 +259,29 @@ describe('localValidationError', () => { expect(err.details).toEqual({ field: 'code-file', reason: 'file does not exist: /tmp/x.ts' }); expect('accepted' in err.details).toBe(false); }); + + // A reason string that already ends in a period (common when a + // call site composes multiple already-punctuated sentences, e.g. + // `test run --all --target-url` builds its rejection out of an + // explanatory sentence + an instruction sentence) must not end up with a + // doubled `..` once the template appends its own trailing period. + it('does not double the trailing period when reason already ends with one', () => { + const err = localValidationError( + 'target-url', + '--target-url has no effect with --all. Remove --target-url.', + ); + expect(err.nextAction).toBe( + 'Flag `--target-url` is invalid: --target-url has no effect with --all. Remove --target-url.', + ); + expect(err.nextAction.endsWith('..')).toBe(false); + expect(err.nextAction.endsWith('.')).toBe(true); + }); + + it('still appends exactly one period when reason has no trailing punctuation', () => { + const err = localValidationError('pageSize', 'must be a positive integer'); + expect(err.nextAction).toBe('Flag `--page-size` is invalid: must be a positive integer.'); + expect(err.nextAction.endsWith('..')).toBe(false); + }); }); describe('ApiError.getDetail', () => { @@ -394,6 +419,32 @@ describe('INSUFFICIENT_CREDITS detection', () => { expect(err.nextAction).not.toContain('https://'); }); + // V3 API keys are membership-bound: a team-org key spends that org's + // wallet, not the personal one. The CLI has no way to know which kind of + // key just failed here (this is a client-side synthesis with no backend + // nextAction), so the synthesized hint must not assert the personal + // billing page is the ONLY fix — it should also point an org-bound caller + // at their org admin instead of silently sending them to top up the wrong + // wallet. + it('synthesized billing hint does not assert a personal-only path (org-bound key wording)', () => { + const err = ApiError.fromEnvelope( + { + error: { + code: 'RATE_LIMITED', + message: 'Insufficient credits: please top up.', + nextAction: '', + requestId: 'req_cred_org', + details: {}, + }, + }, + 429, + ); + expect(err.nextAction.toLowerCase()).toContain('org admin'); + // The personal-key link is still offered — it's a real, correct fix for + // the common case, just no longer presented as the only one. + expect(err.nextAction).toContain('(/dashboard/settings/billing)'); + }); + it('synthesized billing link resolves the PROD portal from a prod apiUrl', () => { const err = ApiError.fromEnvelope( { @@ -508,3 +559,72 @@ describe('INSUFFICIENT_CREDITS detection', () => { expect(err.exitCode).toBe(11); }); }); + +describe('AMBIGUOUS_ORG detection (409 CONFLICT + reason: ambiguous_org)', () => { + it('remaps a CONFLICT envelope with details.reason === "ambiguous_org" to AMBIGUOUS_ORG / exit 6', () => { + const err = ApiError.fromEnvelope( + { + error: { + code: 'CONFLICT', + message: 'Test id "test_x" resolves in more than one of your organizations.', + nextAction: 'Open the specific project in the Portal, or contact support.', + requestId: 'req_ambig_1', + details: { + reason: 'ambiguous_org', + testId: 'test_x', + candidates: [ + { projectId: 'project_a', orgId: 'org_a' }, + { projectId: 'project_b', orgId: 'org_b' }, + ], + }, + }, + }, + 409, + ); + expect(err.code).toBe('AMBIGUOUS_ORG'); + expect(err.exitCode).toBe(6); + // Message/nextAction pass through verbatim — the backend already + // supplies an actionable template, unlike the INSUFFICIENT_CREDITS + // sub-case which sometimes synthesizes one for older backends. + expect(err.message).toContain('resolves in more than one of your organizations'); + expect(err.nextAction).toContain('Open the specific project'); + expect(err.getDetail('testId')).toBe('test_x'); + expect(err.getDetail('candidates')).toEqual([ + { projectId: 'project_a', orgId: 'org_a' }, + { projectId: 'project_b', orgId: 'org_b' }, + ]); + }); + + it('a plain CONFLICT (snapshot in flight, no ambiguous_org reason) stays CONFLICT / exit 6', () => { + const err = ApiError.fromEnvelope( + { + error: { + code: 'CONFLICT', + message: 'Snapshot in flight; retry shortly.', + nextAction: 'Retry in a few seconds.', + requestId: 'req_conflict_1', + details: { reason: 'snapshot_in_flight' }, + }, + }, + 409, + ); + expect(err.code).toBe('CONFLICT'); + expect(err.exitCode).toBe(6); + }); + + it('a CONFLICT with an unrelated reason string stays CONFLICT (not falsely remapped)', () => { + const err = ApiError.fromEnvelope( + { + error: { + code: 'CONFLICT', + message: 'Another run is already in flight.', + nextAction: 'Poll it with test wait.', + requestId: 'req_conflict_2', + details: { reason: 'run_in_flight', currentRunId: 'run_abc' }, + }, + }, + 409, + ); + expect(err.code).toBe('CONFLICT'); + }); +}); diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 40b6c05..f31e7e0 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -26,24 +26,33 @@ export const ERROR_CODES = [ // "a Portal write is in flight." 'IDEMPOTENCY_BODY_MISMATCH', 'RATE_LIMITED', - // Client-side re-map of the credits sub-case of a RATE_LIMITED envelope. - // Detected when the backend returns RATE_LIMITED (HTTP 429) with a - // message matching /insufficient credits/i or details.required present. - // Exit 12: non-retriable — out-of-credits cannot self-heal with retries. - // The genuine per-minute throttle, 60/min/key ("Run trigger rate limit exceeded") - // retains code RATE_LIMITED / exit 11 / retriable. - // Note: backend will emit a distinct code in a future release; this is - // CLI-side detection only, bridging the gap. + // Emitted natively by the backend as HTTP 402 with `details.required` (the + // shortfall) — non-retriable, exit 12: out-of-credits cannot self-heal with + // retries. The CLI additionally re-maps the legacy credits sub-case of a + // RATE_LIMITED envelope (message matching /insufficient credits/i or + // details.required present) for older backends. The genuine per-minute + // throttle, 60/min/key ("Run trigger rate limit exceeded") retains code + // RATE_LIMITED / exit 11 / retriable. 'INSUFFICIENT_CREDITS', - // Client-side code for plan-gated features that the server silently downgrades - // rather than emitting a 4xx. Used when the CLI detects a paid-tier feature was - // requested but the response shows it was not applied (e.g. autoHeal: false when - // autoHeal: true was sent on a Free key). This is NOT thrown as an error by - // default (the command still succeeds with verbatim replay) — it is used by - // callers that want to programmatically detect plan-downgrade events. - // Exit code: 13 (non-retriable). Backend follow-up: emit natively from the server - // so the CLI can drop the client-side detection heuristic. + // Emitted natively by the backend as a fatal HTTP 403 when a plan-gated + // feature is requested by a tier that doesn't include it (e.g. auto-auth on + // a Free-plan org). `details` carries `{feature, plan, reason: 'plan'}`; + // exit code 13 (non-retriable) — the fix is an upgrade, not a retry. + // Distinct from silent tier DOWNGRADES (a request that still succeeds with + // the feature dropped): those are surfaced via the response's advisory + // fields, not this error. 'FEATURE_GATED', + // Client-side re-map of the org-ambiguity sub-case of a CONFLICT (409) + // envelope. Raised when a membership-key caller's testId resolves to + // projects in more than one of their organizations (uuid-derived ids + // should not normally collide, so this is pathological — a shared-id + // union-resolution edge case, not a transient snapshot race). Detected + // when the backend returns `code: "CONFLICT"` with + // `details.reason === "ambiguous_org"`. Same exit-code family as CONFLICT + // (exit 6) but kept as a distinct code — unlike a generic CONFLICT this is + // never retried (retrying resolves the same ambiguity again) and + // `details.candidates[]` needs its own rendering. + 'AMBIGUOUS_ORG', 'UNSUPPORTED', // The running CLI is older than the backend's minimum supported version. // Backend emits this as HTTP 426 when version enforcement is enabled. Exit @@ -80,6 +89,7 @@ export function exitCodeFor(code: ErrorCode): number { case 'CONFLICT': case 'PRECONDITION_FAILED': case 'IDEMPOTENCY_BODY_MISMATCH': + case 'AMBIGUOUS_ORG': return 6; case 'UNSUPPORTED': return 7; @@ -268,6 +278,11 @@ export class ApiError extends CLIError { * * Use `'field'` for JSON-body paths to avoid fabricating a `--fieldName` * flag that doesn't exist in the CLI surface. + * + * `reason` is trailed with a period UNLESS it already ends with one — some + * call sites compose `reason` from multiple already-punctuated sentences + * (e.g. `'... Remove --target-url.'`), and unconditionally appending a + * second period produced a doubled `..` at the end of the rendered message. */ export function localValidationError( field: string, @@ -279,11 +294,12 @@ export function localValidationError( kind === 'flag' ? `Flag \`--${field.replace(/[A-Z]/g, ch => `-${ch.toLowerCase()}`)}\`` : `Field \`${field}\``; + const punctuatedReason = reason.endsWith('.') ? reason : `${reason}.`; return ApiError.fromEnvelope({ error: { code: 'VALIDATION_ERROR', message: 'Invalid request.', - nextAction: `${subject} is invalid: ${reason}.`, + nextAction: `${subject} is invalid: ${punctuatedReason}`, requestId: 'local', details: accepted === undefined ? { field, reason } : { field, reason, accepted }, }, @@ -347,6 +363,8 @@ function codeFromHttpStatus(status: number | undefined): ErrorCode { return 'VALIDATION_ERROR'; case 401: return 'AUTH_INVALID'; + case 402: + return 'INSUFFICIENT_CREDITS'; case 403: return 'AUTH_FORBIDDEN'; case 404: @@ -397,6 +415,17 @@ function isInsufficientCredits( return hasRequiredField || hasCreditsMessage; } +/** + * Detect the "ambiguous org" sub-case of a CONFLICT (409) envelope: a + * membership-key caller's testId resolved to projects in more than one of + * their organizations. `details.reason === "ambiguous_org"` is the backend's + * stable discriminator (see `CliAmbiguousTestError` server-side) — distinct + * from the default CONFLICT meaning (snapshot in flight). + */ +function isAmbiguousOrgConflict(rawCode: string, details: Record): boolean { + return rawCode === 'CONFLICT' && details.reason === 'ambiguous_org'; +} + function parseEnvelopeBody(raw: unknown, httpStatus?: number, apiUrl?: string): ErrorEnvelopeBody { if (typeof raw !== 'object' || raw === null) { const fallbackCode = codeFromHttpStatus(httpStatus); @@ -462,13 +491,22 @@ function parseEnvelopeBody(raw: unknown, httpStatus?: number, apiUrl?: string): // Portal links resolve per environment from the API endpoint (dev and // prod portals live on different domains); unknown hosts get the route // only — a hardcoded domain would point at the wrong environment. + // + // V3 API keys are membership-bound: a key minted for a team org spends + // THAT org's wallet, not the holder's personal one, and there is no + // request-scoped signal here (this branch fires client-side, purely + // from the HTTP status/body) to tell which kind of key just failed. So + // this synthesized hint deliberately does NOT assert "top up your + // (personal) credits" as the only path — it offers the personal-key + // link (still correct for the common case) alongside an honest + // org-bound alternative, rather than guessing. const portalBase = apiUrl === undefined ? undefined : resolvePortalBase(apiUrl); const billingNextAction = nextAction !== '' ? nextAction : (portalBase !== undefined - ? `Top up your credits at ${portalBase}/dashboard/settings/billing or upgrade your plan at ${portalBase}/pricing.` - : 'Top up your credits on the portal Billing page (/dashboard/settings/billing) or upgrade your plan (/pricing).') + + ? `Top up credits at ${portalBase}/dashboard/settings/billing (personal keys) — ask an org admin if this key is organization-bound — or upgrade your plan at ${portalBase}/pricing.` + : 'Top up credits on the portal Billing page (/dashboard/settings/billing) if this is a personal key — ask an org admin if it is organization-bound — or upgrade your plan (/pricing).') + ' Run `testsprite usage` to check your current balance before the next run.'; return { code: 'INSUFFICIENT_CREDITS', @@ -479,6 +517,21 @@ function parseEnvelopeBody(raw: unknown, httpStatus?: number, apiUrl?: string): }; } + // Client-side re-map: CONFLICT with the ambiguous-org discriminator -> + // AMBIGUOUS_ORG. Same message/nextAction as the backend supplied — the + // server already writes an actionable template — but a distinct `code` + // so the retry decision (http.ts) and the candidates rendering (index.ts) + // can key off it directly instead of re-parsing `details.reason`. + if (isAmbiguousOrgConflict(rawCode, details)) { + return { + code: 'AMBIGUOUS_ORG', + message, + nextAction, + requestId, + details, + }; + } + return { code: rawCode, message, diff --git a/src/lib/flaky.test.ts b/src/lib/flaky.test.ts index a6acd9c..ed7ac7a 100644 --- a/src/lib/flaky.test.ts +++ b/src/lib/flaky.test.ts @@ -79,6 +79,17 @@ describe('summarizeFlaky', () => { expect(report.runs).toBe(2); expect(report.verdict).toBe('flaky'); }); + + it('defaults advisories to an empty array when the orchestrator passes none', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2)]); + expect(report.advisories).toEqual([]); + }); + + it('carries a deduped advisories list through untouched', () => { + const advisories = [{ feature: 'autoHeal', message: 'not yet enforced' }]; + const report = summarizeFlaky('test_x', [pass(1), pass(2)], advisories); + expect(report.advisories).toEqual(advisories); + }); }); describe('renderFlakyText', () => { @@ -102,4 +113,20 @@ describe('renderFlakyText', () => { ); expect(text).toContain('#1 (no runId) error'); }); + + it('renders one [advisory] line per entry when the report carries advisories', () => { + const text = renderFlakyText( + summarizeFlaky( + 'test_login', + [pass(1), pass(2)], + [{ feature: 'autoHeal', message: 'not yet enforced' }], + ), + ); + expect(text).toContain('[advisory] not yet enforced'); + }); + + it('prints no advisory line when the report carries none', () => { + const text = renderFlakyText(summarizeFlaky('test_login', [pass(1), pass(2)])); + expect(text).not.toContain('[advisory]'); + }); }); diff --git a/src/lib/flaky.ts b/src/lib/flaky.ts index 402eaf3..0c1e95d 100644 --- a/src/lib/flaky.ts +++ b/src/lib/flaky.ts @@ -8,6 +8,7 @@ * (deterministic, no network / credentials), matching the repo's mock-based * test convention. */ +import type { RerunAdvisory } from './runs.types.js'; /** * Outcome of a single flaky-detector attempt. The first four mirror the @@ -60,6 +61,16 @@ export interface FlakyReport { verdict: FlakyVerdict; /** Non-passing attempts, in attempt order. */ failures: FlakyFailure[]; + /** + * Server-side advisories collected across all replay attempts and deduped + * (by `feature`+`message`) — surfaced ONCE per probe, not once per attempt. + * Empty when the server sent none, which is the common case: absent on + * every V2 response and every V3 response that isn't a V3-routed rerun + * with an explicit `autoHeal:false` opt-out (see `RerunAdvisory`). `test + * flaky` always sends `autoHeal:false`, so this is populated whenever the + * probe's reruns are routed to V3. + */ + advisories: RerunAdvisory[]; } /** @@ -72,7 +83,11 @@ export interface FlakyReport { * An empty attempt list (no runs observed) is reported as `failing` with a * `0` ratio — there is no evidence the test is stable. */ -export function summarizeFlaky(testId: string, attempts: FlakyAttempt[]): FlakyReport { +export function summarizeFlaky( + testId: string, + attempts: FlakyAttempt[], + advisories: RerunAdvisory[] = [], +): FlakyReport { const runs = attempts.length; const passed = attempts.filter(a => a.outcome === 'passed').length; const failed = runs - passed; @@ -87,7 +102,7 @@ export function summarizeFlaky(testId: string, attempts: FlakyAttempt[]): FlakyR outcome: a.outcome, failureKind: a.failureKind ?? null, })); - return { testId, runs, passed, failed, stableRatio, verdict, failures }; + return { testId, runs, passed, failed, stableRatio, verdict, failures, advisories }; } /** @@ -129,5 +144,10 @@ export function renderFlakyText(report: FlakyReport): string { lines.push(` #${f.attempt} ${rid} ${f.outcome}${kind}`); } } + if (report.advisories.length > 0) { + for (const advisory of report.advisories) { + lines.push(` [advisory] ${advisory.message}`); + } + } return lines.join('\n'); } diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index e8f4c35..0626ee1 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -280,6 +280,34 @@ describe('HttpClient error mapping', () => { expect(fetchImpl).toHaveBeenCalledTimes(2); }); + it('does not retry a CONFLICT remapped to AMBIGUOUS_ORG (permanent id collision, not a race)', async () => { + const fetchImpl = vi.fn(async () => + jsonResponse( + { + error: { + code: 'CONFLICT', + message: 'Test id "test_x" resolves in more than one of your organizations.', + nextAction: 'Open the specific project in the Portal, or contact support.', + requestId: 'req_ambig', + details: { + reason: 'ambiguous_org', + testId: 'test_x', + candidates: [{ projectId: 'p1', orgId: 'o1' }], + }, + }, + }, + { status: 409 }, + ), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch); + const err = await client.get('/tests/test_x').catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('AMBIGUOUS_ORG'); + expect((err as ApiError).exitCode).toBe(6); + expect((err as ApiError).getDetail('candidates')).toEqual([{ projectId: 'p1', orgId: 'o1' }]); + expect(fetchImpl).toHaveBeenCalledTimes(1); // no retry — distinct from generic CONFLICT + }); + it('retries INTERNAL once then propagates', async () => { const fetchImpl = vi.fn(async () => errorEnvelopeResponse(500, 'INTERNAL')); const client = makeClient(fetchImpl as unknown as typeof fetch); diff --git a/src/lib/http.ts b/src/lib/http.ts index 34a58ec..65b102a 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -354,8 +354,9 @@ export class HttpClient { /** * POST /api/cli/v1/tests/{testId}/runs/rerun - * Trigger a rerun (replay) for a single test. FE: verbatim script replay (no credits). - * BE: dependency-closure re-run. Returns `runId` + optional `closure` (BE). + * Trigger a rerun (replay) for a single test. FE: verbatim script replay, billed at + * 0.5 credits (same as a fresh run; legacy V2 accounts: free). BE: dependency-closure + * re-run, billed at 0.2 credits. Returns `runId` + optional `closure` (BE). * * `retryOnConflict: false` — 409 on rerun means the test is already in-flight, * a persistent condition. Retrying would race against the running test. @@ -1055,6 +1056,9 @@ function apiRetryDecision( // case below and retries normally. // FEATURE_GATED is non-retriable: a paid-feature gate can't self-heal with // retries — the caller must upgrade their plan first. + // AMBIGUOUS_ORG is non-retriable: unlike a generic CONFLICT (mid-mutation + // snapshot race), this is a permanent id collision across the caller's + // organizations — retrying resolves the exact same ambiguity again. case 'AUTH_REQUIRED': case 'AUTH_INVALID': case 'AUTH_FORBIDDEN': @@ -1067,6 +1071,7 @@ function apiRetryDecision( case 'INSUFFICIENT_CREDITS': case 'FEATURE_GATED': case 'CLIENT_TOO_OLD': + case 'AMBIGUOUS_ORG': // CLIENT_TOO_OLD: retrying re-sends the same too-old client — it can only // self-heal by upgrading, so fail fast with the upgrade guidance. return { retry: false, delayMs: 0 }; diff --git a/src/lib/org-render.test.ts b/src/lib/org-render.test.ts new file mode 100644 index 0000000..1799255 --- /dev/null +++ b/src/lib/org-render.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest'; +import { formatOrgBinding, formatOrgsSummary, formatPersonalScopeHint } from './org-render.js'; + +describe('formatOrgsSummary', () => { + it('returns undefined when organizations is undefined', () => { + expect(formatOrgsSummary(undefined)).toBeUndefined(); + }); + + it('returns undefined when organizations is an empty array', () => { + expect(formatOrgsSummary([])).toBeUndefined(); + }); + + it('renders a single organization', () => { + const out = formatOrgsSummary([ + { id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }, + ]); + expect(out).toBe('Acme Corp (org_1, role: owner)'); + }); + + it('renders multiple organizations, semicolon-separated', () => { + const out = formatOrgsSummary([ + { id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }, + { id: 'org_2', name: "Jane's workspace", role: 'member', isPersonal: true }, + ]); + expect(out).toBe( + "Acme Corp (org_1, role: owner); Jane's workspace (org_2, personal, role: member)", + ); + }); + + it('marks the personal org distinctly', () => { + const out = formatOrgsSummary([ + { id: 'org_personal', name: "Jane's workspace", role: 'owner', isPersonal: true }, + ]); + expect(out).toContain(', personal, role: owner'); + }); +}); + +describe('formatOrgBinding', () => { + it('returns undefined when org is undefined (legacy key, or older backend)', () => { + expect(formatOrgBinding(undefined)).toBeUndefined(); + }); + + it('renders the binding with a resolved name', () => { + const out = formatOrgBinding({ id: 'org_1', name: 'Acme Corp', role: 'member' }); + expect(out).toBe('Acme Corp (org_1, role: member)'); + }); + + it('falls back to the id when name resolution failed (name: null)', () => { + const out = formatOrgBinding({ id: 'org_1', name: null, role: 'member' }); + expect(out).toBe('org_1 (org_1, role: member)'); + }); +}); + +describe('formatPersonalScopeHint', () => { + const personal = { id: 'org-p', name: 'Duke', role: 'owner', isPersonal: true }; + const team = { id: 'org-t', name: 'Acme', role: 'member', isPersonal: false }; + + it('names the workspaces an unbound key cannot reach', () => { + const hint = formatPersonalScopeHint([personal, team], undefined); + expect(hint).toContain('Acme'); + expect(hint).toContain('Settings → API Keys'); + }); + + it('says nothing when the key is already workspace-bound', () => { + expect( + formatPersonalScopeHint([personal, team], { id: 'org-t', name: 'Acme', role: 'member' }), + ).toBeUndefined(); + }); + + it('says nothing for a solo user — there is no other workspace to miss', () => { + expect(formatPersonalScopeHint([personal], undefined)).toBeUndefined(); + }); + + it('is absent-safe when the backend omits organizations', () => { + expect(formatPersonalScopeHint(undefined, undefined)).toBeUndefined(); + }); + + it('lists every team workspace, not just the first', () => { + const hint = formatPersonalScopeHint( + [personal, team, { id: 'org-u', name: 'Globex', role: 'admin', isPersonal: false }], + undefined, + ); + expect(hint).toContain('Acme, Globex'); + }); +}); diff --git a/src/lib/org-render.ts b/src/lib/org-render.ts new file mode 100644 index 0000000..130770f --- /dev/null +++ b/src/lib/org-render.ts @@ -0,0 +1,80 @@ +/** + * Shared org-attribution text rendering for `auth status`, `usage`, and + * `doctor` — every surface that reads `GET /me`. + * + * `organizations[]` (the caller's full account-wide membership list) and + * `org` (a membership key's own org binding, present only for a + * Postgres-backed `sk-member-…` key) are both optional/absent-safe on the `/me` + * response: older backends omit them entirely, and a lookup hiccup + * server-side omits `organizations` without failing the request. Callers + * render a line only when the corresponding formatter returns a non-undefined + * string — never an `undefined`/`null` literal. + */ + +/** One organization from `MeResponse.organizations[]`. */ +export interface CliOrgSummary { + id: string; + name: string; + role: string; + isPersonal: boolean; +} + +/** A membership key's own org binding, from `MeResponse.org`. */ +export interface CliOrgBinding { + id: string; + /** `null` when best-effort name resolution failed or found no match server-side. */ + name: string | null; + role: string; +} + +/** + * One-line summary of the caller's full membership list, for an `orgs:` + * line. Returns `undefined` when the list is absent or empty so the caller + * can omit the line entirely. + */ +export function formatOrgsSummary( + organizations: readonly CliOrgSummary[] | undefined, +): string | undefined { + if (!organizations || organizations.length === 0) return undefined; + return organizations + .map(o => `${o.name} (${o.id}${o.isPersonal ? ', personal' : ''}, role: ${o.role})`) + .join('; '); +} + +/** + * One-line summary of a membership key's own org binding, for an + * `org binding:` line. Returns `undefined` when no binding is present + * (legacy envelope key, or a backend that doesn't return it). + */ +export function formatOrgBinding(org: CliOrgBinding | undefined): string | undefined { + if (!org) return undefined; + const label = org.name ?? org.id; + return `${label} (${org.id}, role: ${org.role})`; +} + +/** + * Hint for a caller whose key can only reach their personal workspace while + * they are a member of at least one team workspace. + * + * A key is bound to exactly ONE membership — the workspace is chosen when the + * key is minted, not per command, which is why there is no `--org` flag to + * reach for. Without this line the failure mode is silent and baffling: the + * team's projects simply don't appear in `project list`, and addressing one by + * id 404s exactly like a typo would. + * + * Returns `undefined` when the key is already workspace-bound, or when the + * caller has no team workspace to be confused about. + */ +export function formatPersonalScopeHint( + organizations: readonly CliOrgSummary[] | undefined, + org: CliOrgBinding | undefined, +): string | undefined { + if (org) return undefined; + const teams = (organizations ?? []).filter(o => !o.isPersonal); + if (teams.length === 0) return undefined; + const names = teams.map(o => o.name).join(', '); + return ( + `this key is scoped to your personal workspace, so nothing in ${names} is reachable with it. ` + + `A key belongs to one workspace: mint one from that workspace's Settings → API Keys and use it here.` + ); +} diff --git a/src/lib/plan-schema.spec.ts b/src/lib/plan-schema.spec.ts new file mode 100644 index 0000000..9559755 --- /dev/null +++ b/src/lib/plan-schema.spec.ts @@ -0,0 +1,216 @@ +/** + * `schemas/plan.schema.json` conformance tests. + * + * Guards the "three surfaces, one schema" contract: + * + * 1. The schema file is itself valid JSON Schema (ajv can compile it). + * 2. The schema accepts/rejects a handful of fixture plans EXACTLY the way + * the real validator (`assertPlanShape`, reached here through the + * public `runCreateFromPlan` entry point — the same one `test create + * --plan-from` uses) accepts/rejects them. If the two ever disagree, + * that's a drift bug this test is designed to catch. + * 3. `PLAN_TEMPLATE_WITH_SCHEMA` (== `test create --plan-template`'s + * stdout, == the example embedded in `test create --help`) validates + * against the schema. + * + * Fixtures mirror the four scenarios: (1) top-level array, (2) + * steps nested under `plan.steps`, (3) missing `projectId`, and (4) a + * `{{...}}` placeholder — which is intentionally VALID (advisory-only, not + * a shape violation) per both the schema and the validator. + */ +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Ajv, type ValidateFunction } from 'ajv'; +import { describe, expect, it, beforeAll } from 'vitest'; +import { + PLAN_SCHEMA_URL, + PLAN_TEMPLATE_TEXT, + PLAN_TEMPLATE_WITH_SCHEMA, + runCreateFromPlan, +} from '../commands/test.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const SCHEMA_PATH = join(__dirname, '..', '..', 'schemas', 'plan.schema.json'); + +function writePlanFile(dir: string, plan: unknown): string { + const path = join(dir, `plan-${Math.random().toString(36).slice(2)}.json`); + writeFileSync(path, typeof plan === 'string' ? plan : JSON.stringify(plan), 'utf8'); + return path; +} + +/** + * Runs the fixture through the SAME validator `test create --plan-from` + * uses (`assertPlanShape`, reached via the public `runCreateFromPlan` + * entry point — `assertPlanShape` itself is module-private). `--dry-run` + * still runs the full local validation pass while making no network calls and needing no credentials + * file, so this is reproducible with zero ambient environment state. + */ +async function passesRealValidator(dir: string, plan: unknown): Promise { + const planFile = writePlanFile(dir, plan); + try { + await runCreateFromPlan( + { + profile: 'default', + output: 'json', + debug: false, + planFrom: planFile, + dryRun: true, + endpointUrl: 'https://api.testsprite.com', + }, + { stdout: () => undefined, stderr: () => undefined }, + ); + return true; + } catch { + return false; + } +} + +describe('schemas/plan.schema.json', () => { + let schema: unknown; + let validate: ValidateFunction; + let dir: string; + + beforeAll(() => { + schema = JSON.parse(readFileSync(SCHEMA_PATH, 'utf8')); + // strict: false — the schema uses a `description` keyword on every node + // (documentation, not validation) which ajv's strict mode otherwise + // warns about; allErrors surfaces every violation for easier debugging. + const ajv = new Ajv({ allErrors: true, strict: false }); + validate = ajv.compile(schema as object); + dir = mkdtempSync(join(tmpdir(), 'cli-plan-schema-')); + }); + + it('is itself valid JSON Schema (ajv compiles it without throwing)', () => { + expect(schema).toBeTruthy(); + expect(typeof validate).toBe('function'); + }); + + // The schema's own `$id` (an identity, not a + // fetch instruction) intentionally stays pinned to the canonical `main` + // URL, while `PLAN_SCHEMA_URL` (embedded in generated plan files as the + // `$schema` fetch hint) is intentionally version-pinned to the running + // CLI's `v` tag — a plan authored against this CLI version should + // always resolve the SAME schema, not whatever `main` contains later. The + // two are deliberately DIFFERENT values; this test locks in that they stay + // on their respective branch/tag conventions rather than accidentally + // converging (or diverging further) on a future edit. + it('`$id` stays pinned to `main` (identity) while `PLAN_SCHEMA_URL` is pinned to a version tag (fetch hint) — see PLAN_SCHEMA_URL doc comment', () => { + const id = (schema as { $id?: string }).$id; + expect(id).toBe( + 'https://raw.githubusercontent.com/TestSprite/testsprite-cli/main/schemas/plan.schema.json', + ); + expect(PLAN_SCHEMA_URL).toMatch( + /^https:\/\/raw\.githubusercontent\.com\/TestSprite\/testsprite-cli\/v\d+\.\d+\.\d+\/schemas\/plan\.schema\.json$/, + ); + expect(PLAN_SCHEMA_URL).not.toBe(id); + }); + + it('accepts the canonical plan template (also asserted against the real validator)', async () => { + expect(validate(PLAN_TEMPLATE_WITH_SCHEMA)).toBe(true); + expect(await passesRealValidator(dir, PLAN_TEMPLATE_WITH_SCHEMA)).toBe(true); + }); + + it('PLAN_TEMPLATE_TEXT is exactly JSON.stringify(PLAN_TEMPLATE_WITH_SCHEMA, null, 2) — no hand-formatted drift', () => { + expect(PLAN_TEMPLATE_TEXT).toBe(JSON.stringify(PLAN_TEMPLATE_WITH_SCHEMA, null, 2)); + // And the rendered text must itself round-trip through JSON.parse + the schema. + expect(validate(JSON.parse(PLAN_TEMPLATE_TEXT))).toBe(true); + }); + + it('accepts a plan exercising every optional field (description, priority)', async () => { + const plan = { + projectId: 'prj_abc123', + type: 'frontend', + name: 'Full-featured plan', + description: 'Exercises every optional field.', + priority: 'p0', + planSteps: [ + { type: 'action', description: 'do a thing' }, + { type: 'assertion', description: 'verify a thing' }, + ], + }; + expect(validate(plan)).toBe(true); + expect(await passesRealValidator(dir, plan)).toBe(true); + }); + + it('rejects type: "backend" — schema is the ground truth for the --plan-from COMMAND, which rejects backend end-to-end (both sides must agree)', async () => { + const plan = { ...PLAN_TEMPLATE_WITH_SCHEMA, type: 'backend' }; + expect(validate(plan)).toBe(false); + expect(await passesRealValidator(dir, plan)).toBe(false); + }); + + it('accepts a non-string `$schema` — the CLI ignores it entirely (assertPlanShape has no additionalProperties check), so the schema leaves it type-unconstrained to match (both sides must agree)', async () => { + const plan = { ...PLAN_TEMPLATE_WITH_SCHEMA, $schema: 123 }; + expect(validate(plan)).toBe(true); + expect(await passesRealValidator(dir, plan)).toBe(true); + }); + + // -- fixture #1: top-level array ----------------------------------- + it('rejects a top-level array (matches the real validator: "must be a JSON object")', async () => { + const plan = [PLAN_TEMPLATE_WITH_SCHEMA]; + expect(validate(plan)).toBe(false); + expect(await passesRealValidator(dir, plan)).toBe(false); + }); + + // -- fixture #2: steps nested under plan.steps --------------------- + it('rejects steps nested under `plan.steps` (missing top-level planSteps; matches the real validator)', async () => { + const plan = { + projectId: 'prj_abc123', + type: 'frontend', + name: 'x', + plan: { steps: [{ type: 'action', description: 'go' }] }, + }; + expect(validate(plan)).toBe(false); + expect(await passesRealValidator(dir, plan)).toBe(false); + }); + + // -- fixture #3: missing projectId ---------------------------------- + it('rejects a plan missing projectId (matches the real validator)', async () => { + const plan = { + type: 'frontend', + name: 'x', + planSteps: [{ type: 'action', description: 'go' }], + }; + expect(validate(plan)).toBe(false); + expect(await passesRealValidator(dir, plan)).toBe(false); + }); + + // -- fixture #4: {{placeholder}} — VALID shape, advisory-only ------ + it('accepts a plan with a `{{VAR}}` placeholder step (structurally valid — the CLI advisory is a separate content-quality check)', async () => { + const plan = { + ...PLAN_TEMPLATE_WITH_SCHEMA, + planSteps: [{ type: 'action', description: 'log in as {{LOGIN_USER}}' }], + }; + expect(validate(plan)).toBe(true); + expect(await passesRealValidator(dir, plan)).toBe(true); + }); + + it('rejects an empty planSteps array (min 1) and one exceeding 200 items (max)', () => { + expect(validate({ ...PLAN_TEMPLATE_WITH_SCHEMA, planSteps: [] })).toBe(false); + const tooMany = Array.from({ length: 201 }, (_, i) => ({ + type: 'action' as const, + description: `step ${i}`, + })); + expect(validate({ ...PLAN_TEMPLATE_WITH_SCHEMA, planSteps: tooMany })).toBe(false); + }); + + it('rejects an invalid planSteps[].type and an invalid top-level priority', () => { + expect( + validate({ + ...PLAN_TEMPLATE_WITH_SCHEMA, + planSteps: [{ type: 'click', description: 'x' }], + }), + ).toBe(false); + expect(validate({ ...PLAN_TEMPLATE_WITH_SCHEMA, priority: 'p9' })).toBe(false); + }); + + it('rejects a whitespace-only projectId/name (pattern requires a non-whitespace character)', () => { + expect(validate({ ...PLAN_TEMPLATE_WITH_SCHEMA, projectId: ' ' })).toBe(false); + expect(validate({ ...PLAN_TEMPLATE_WITH_SCHEMA, name: ' ' })).toBe(false); + }); + + it('allows unknown extra top-level properties (matches assertPlanShape, which has no additionalProperties check)', () => { + expect(validate({ ...PLAN_TEMPLATE_WITH_SCHEMA, someFutureField: 'anything' })).toBe(true); + }); +}); diff --git a/src/lib/poll.ts b/src/lib/poll.ts index 92498e2..94b3ae4 100644 --- a/src/lib/poll.ts +++ b/src/lib/poll.ts @@ -294,7 +294,7 @@ async function pollLoop( shutdownSignal != null ? AbortSignal.any([altAbort.signal, shutdownSignal]) : altAbort.signal; - let alternate: RunResponse | null = null; + let alternate: RunResponse | null; try { alternate = await resolveAlternate(run, elapsedMs, altSignal); } finally { diff --git a/src/lib/render-error.test.ts b/src/lib/render-error.test.ts index 9635f2b..5b679c2 100644 --- a/src/lib/render-error.test.ts +++ b/src/lib/render-error.test.ts @@ -7,7 +7,11 @@ * subcommand name. */ import { describe, expect, it } from 'vitest'; -import { renderCommanderError, rephraseUnknownOption } from './render-error.js'; +import { + renderAmbiguousOrgCandidates, + renderCommanderError, + rephraseUnknownOption, +} from './render-error.js'; describe('rephraseUnknownOption', () => { it('rephrases --dry-run placed after subcommand', () => { @@ -157,3 +161,59 @@ describe('renderCommanderError', () => { expect(Object.keys(parsed)).toEqual(['error']); }); }); + +describe('renderAmbiguousOrgCandidates', () => { + it('renders one candidate line per well-formed entry, plus a --project hint', () => { + const lines = renderAmbiguousOrgCandidates([ + { projectId: 'project_a', orgId: 'org_a' }, + { projectId: 'project_b', orgId: 'org_b' }, + ]); + expect(lines).toEqual([ + ' candidate: project project_a (org org_a)', + ' candidate: project project_b (org org_b)', + ' hint: re-run with --project to disambiguate.', + ]); + }); + + it('renders a single candidate', () => { + const lines = renderAmbiguousOrgCandidates([{ projectId: 'p1', orgId: 'o1' }]); + expect(lines).toEqual([ + ' candidate: project p1 (org o1)', + ' hint: re-run with --project to disambiguate.', + ]); + }); + + it('returns an empty array for undefined', () => { + expect(renderAmbiguousOrgCandidates(undefined)).toEqual([]); + }); + + it('returns an empty array for a non-array value', () => { + expect(renderAmbiguousOrgCandidates('not-an-array')).toEqual([]); + expect(renderAmbiguousOrgCandidates({ projectId: 'p1', orgId: 'o1' })).toEqual([]); + expect(renderAmbiguousOrgCandidates(null)).toEqual([]); + }); + + it('returns an empty array for an empty array', () => { + expect(renderAmbiguousOrgCandidates([])).toEqual([]); + }); + + it('skips malformed entries (missing projectId/orgId) without throwing', () => { + const lines = renderAmbiguousOrgCandidates([ + { projectId: 'project_a' }, // missing orgId + { orgId: 'org_b' }, // missing projectId + null, + 'not-an-object', + 42, + { projectId: 'project_c', orgId: 'org_c' }, + ]); + expect(lines).toEqual([ + ' candidate: project project_c (org org_c)', + ' hint: re-run with --project to disambiguate.', + ]); + }); + + it('returns an empty array when every entry is malformed (no trailing hint either)', () => { + const lines = renderAmbiguousOrgCandidates([{ projectId: 123, orgId: 'org_a' }, {}]); + expect(lines).toEqual([]); + }); +}); diff --git a/src/lib/render-error.ts b/src/lib/render-error.ts index 1898acd..5387fd5 100644 --- a/src/lib/render-error.ts +++ b/src/lib/render-error.ts @@ -70,6 +70,33 @@ export function renderCommanderError( return pendingMsg ?? `${rawMsg}\n`; } +/** + * Text-mode rendering for the `AMBIGUOUS_ORG` conflict (`details.candidates[]` + * — an array of `{ projectId, orgId }` pairs the same testId resolved to, + * across the caller's organizations). Renders one actionable `candidate:` + * line per well-formed entry, plus a trailing hint to disambiguate with + * `--project `. + * + * Defensive by design: any entry missing `projectId`/`orgId` (or a + * malformed/empty/absent `candidates` value) is silently skipped rather than + * thrown — this is best-effort stderr decoration on top of an error that + * already renders correctly via the generic envelope, so a future server + * shape change must never crash the CLI's error path. + */ +export function renderAmbiguousOrgCandidates(candidates: unknown): string[] { + if (!Array.isArray(candidates)) return []; + const lines: string[] = []; + for (const candidate of candidates) { + if (candidate === null || typeof candidate !== 'object') continue; + const { projectId, orgId } = candidate as { projectId?: unknown; orgId?: unknown }; + if (typeof projectId !== 'string' || typeof orgId !== 'string') continue; + lines.push(` candidate: project ${projectId} (org ${orgId})`); + } + if (lines.length === 0) return []; + lines.push(' hint: re-run with --project to disambiguate.'); + return lines; +} + export function rephraseUnknownOption(raw: string): string | null { // Commander emits: "error: unknown option '--foo'" const match = /unknown option\s+'--([^']+)'/.exec(raw); diff --git a/src/lib/response-schemas.test.ts b/src/lib/response-schemas.test.ts index 79c56d6..507fbaf 100644 --- a/src/lib/response-schemas.test.ts +++ b/src/lib/response-schemas.test.ts @@ -7,7 +7,14 @@ import { describe, expect, it } from 'vitest'; import * as v from 'valibot'; import { HttpClient } from './http.js'; -import { RUN_RESPONSE_SCHEMA, TRIGGER_RUN_RESPONSE_SCHEMA } from './response-schemas.js'; +import { + BATCH_RERUN_RESPONSE_SCHEMA, + LIST_RUNS_RESPONSE_SCHEMA, + ME_IDENTITY_SCHEMA, + RERUN_RESPONSE_SCHEMA, + RUN_RESPONSE_SCHEMA, + TRIGGER_RUN_RESPONSE_SCHEMA, +} from './response-schemas.js'; const VALID_RUN = { runId: 'run_1', @@ -60,6 +67,72 @@ describe('RUN_RESPONSE_SCHEMA', () => { expect(parsed.issues.some(issue => v.getDotPath(issue) === 'status')).toBe(true); } }); + + it('accepts targetUrl: null and keeps it null (backend runs, and any run whose engine records no URL)', () => { + const parsed = v.safeParse(RUN_RESPONSE_SCHEMA, { ...VALID_RUN, targetUrl: null }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.output.targetUrl).toBeNull(); + } + }); + + it('accepts codeVersion: null (pre-M3.1 rows, and tests with no stored code body)', () => { + const parsed = v.safeParse(RUN_RESPONSE_SCHEMA, { ...VALID_RUN, codeVersion: null }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.output.codeVersion).toBeNull(); + } + }); + + it('normalizes an omitted targetUrl / codeVersion to null', () => { + const withoutBoth: Record = { ...VALID_RUN }; + delete withoutBoth.targetUrl; + delete withoutBoth.codeVersion; + const parsed = v.safeParse(RUN_RESPONSE_SCHEMA, withoutBoth); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.output.targetUrl).toBeNull(); + expect(parsed.output.codeVersion).toBeNull(); + } + }); + + // The three cases below pin the whole `dashboardUrl` contract at the parse + // boundary — the only place it can be broken invisibly, since the command + // tests build `RunResponse` objects by hand and never cross this schema. + it('leaves an omitted dashboardUrl as an ABSENT key, not a materialized null (keeps the client fallback alive)', () => { + const parsed = v.safeParse(RUN_RESPONSE_SCHEMA, { ...VALID_RUN }); + expect(parsed.success).toBe(true); + if (parsed.success) { + // `withRunDashboardUrl` branches on `'dashboardUrl' in run`: an absent key + // means "this backend sends no link, compute one myself". Switching the + // schema default from `undefined` to `null` (matching the fields above) + // would make this key always present and turn that fallback into dead + // code for every older backend. + expect('dashboardUrl' in parsed.output).toBe(false); + expect(Object.keys(parsed.output)).not.toContain('dashboardUrl'); + } + }); + + it('accepts dashboardUrl: null without failing validation (a null must never take down `test wait`)', () => { + const parsed = v.safeParse(RUN_RESPONSE_SCHEMA, { ...VALID_RUN, dashboardUrl: null }); + expect(parsed.success).toBe(true); + if (parsed.success) { + // Preserved as null, not coerced: the default applies to an ABSENT key + // only. Consumers normalize (a null suppresses the link and does NOT + // substitute the client guess). + expect('dashboardUrl' in parsed.output).toBe(true); + expect(parsed.output.dashboardUrl).toBeNull(); + } + }); + + it('accepts and preserves a server-sent dashboardUrl string', () => { + const link = 'https://portal.example.com/dashboard-v3/o/org_1/projects/p_1/test-cases/test_1'; + const parsed = v.safeParse(RUN_RESPONSE_SCHEMA, { ...VALID_RUN, dashboardUrl: link }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.output.dashboardUrl).toBe(link); + } + }); }); describe('HttpClient schema hook', () => { @@ -79,6 +152,19 @@ describe('HttpClient schema hook', () => { expect(JSON.stringify(issues)).toContain('status'); }); + it('getRun resolves normally when the run carries targetUrl: null (never an INTERNAL envelope)', async () => { + const fetchImpl = (async () => + new Response(JSON.stringify({ ...VALID_RUN, targetUrl: null, codeVersion: null }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })) as typeof fetch; + const client = makeClient(fetchImpl); + const run = await client.getRun('run_1'); + expect(run.status).toBe('passed'); + expect(run.targetUrl).toBeNull(); + expect(run.codeVersion).toBeNull(); + }); + it('a schemaless generic get still returns whatever JSON came back (unchanged behavior)', async () => { const fetchImpl = (async () => new Response(JSON.stringify({ anything: true }), { @@ -102,3 +188,166 @@ describe('TRIGGER_RUN_RESPONSE_SCHEMA', () => { expect(parsed.success).toBe(true); }); }); + +describe('LIST_RUNS_RESPONSE_SCHEMA', () => { + const HISTORY_ROW = { + runId: 'run_1', + status: 'passed', + source: 'cli', + isRerun: false, + createdFrom: null, + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: null, + finishedAt: '2026-06-01T10:00:30.000Z', + codeVersion: 'v1', + failureKind: null, + }; + + it('accepts a history row with codeVersion: null (pre-M3.2 rows, tests with no code body)', () => { + const parsed = v.safeParse(LIST_RUNS_RESPONSE_SCHEMA, { + runs: [{ ...HISTORY_ROW, codeVersion: null }], + nextCursor: null, + meta: {}, + }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.output.runs[0]?.codeVersion).toBeNull(); + } + }); + + it('accepts a history row with targetUrl: null + targetUrlSource: null', () => { + const parsed = v.safeParse(LIST_RUNS_RESPONSE_SCHEMA, { + runs: [{ ...HISTORY_ROW, targetUrl: null, targetUrlSource: null }], + nextCursor: null, + meta: {}, + }); + expect(parsed.success).toBe(true); + }); +}); + +const VALID_RERUN = { + runId: 'run_rerun_1', + status: 'queued', + enqueuedAt: '2026-06-01T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, +}; + +describe('RERUN_RESPONSE_SCHEMA — advisories (optional additive field)', () => { + it('accepts a response with no advisories field (every V2 response, older backends)', () => { + const parsed = v.safeParse(RERUN_RESPONSE_SCHEMA, VALID_RERUN); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect((parsed.output as { advisories?: unknown }).advisories).toBeUndefined(); + } + }); + + it('accepts and preserves advisories when present (V3-routed rerun with autoHeal:false)', () => { + const parsed = v.safeParse(RERUN_RESPONSE_SCHEMA, { + ...VALID_RERUN, + advisories: [ + { + feature: 'autoHeal', + message: + 'The auto-heal opt-out was forwarded to the execution engine but is not yet enforced there.', + }, + ], + }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.output.advisories).toEqual([ + { + feature: 'autoHeal', + message: + 'The auto-heal opt-out was forwarded to the execution engine but is not yet enforced there.', + }, + ]); + } + }); + + it('accepts an empty advisories array', () => { + const parsed = v.safeParse(RERUN_RESPONSE_SCHEMA, { ...VALID_RERUN, advisories: [] }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.output.advisories).toEqual([]); + } + }); +}); + +const VALID_BATCH_RERUN = { + accepted: [{ testId: 'test_1', runId: 'run_1', enqueuedAt: '2026-06-01T10:00:00.000Z' }], + deferred: [], + conflicts: [], + closure: { byProject: [] }, +}; + +describe('BATCH_RERUN_RESPONSE_SCHEMA — advisories (optional additive field)', () => { + it('accepts a response with no advisories field', () => { + const parsed = v.safeParse(BATCH_RERUN_RESPONSE_SCHEMA, VALID_BATCH_RERUN); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect((parsed.output as { advisories?: unknown }).advisories).toBeUndefined(); + } + }); + + it('accepts and preserves advisories when present', () => { + const parsed = v.safeParse(BATCH_RERUN_RESPONSE_SCHEMA, { + ...VALID_BATCH_RERUN, + advisories: [{ feature: 'autoHeal', message: 'not yet enforced' }], + }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.output.advisories).toEqual([ + { feature: 'autoHeal', message: 'not yet enforced' }, + ]); + } + }); +}); + +describe('ME_IDENTITY_SCHEMA', () => { + it('accepts a bare identity core with no org fields (older backend)', () => { + const parsed = v.safeParse(ME_IDENTITY_SCHEMA, { userId: 'u_1', keyId: 'k_1' }); + expect(parsed.success).toBe(true); + }); + + it('accepts organizations[] and org together (membership key)', () => { + const parsed = v.safeParse(ME_IDENTITY_SCHEMA, { + userId: 'u_1', + keyId: 'k_1', + organizations: [ + { id: 'org_1', name: 'Acme Corp', role: 'owner', isPersonal: false }, + { id: 'org_2', name: "u_1's workspace", role: 'owner', isPersonal: true }, + ], + org: { id: 'org_1', name: 'Acme Corp', role: 'owner' }, + }); + expect(parsed.success).toBe(true); + }); + + it('accepts org.name === null (best-effort resolution failed server-side)', () => { + const parsed = v.safeParse(ME_IDENTITY_SCHEMA, { + org: { id: 'org_1', name: null, role: 'member' }, + }); + expect(parsed.success).toBe(true); + }); + + it('preserves unknown extra keys on the full /me projection (loose object)', () => { + const parsed = v.safeParse(ME_IDENTITY_SCHEMA, { + userId: 'u_1', + keyId: 'k_1', + scopes: ['read:me'], + env: 'development', + v3Enabled: true, + }); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect((parsed.output as { v3Enabled?: boolean }).v3Enabled).toBe(true); + } + }); + + it('rejects a malformed organizations entry (missing role)', () => { + const parsed = v.safeParse(ME_IDENTITY_SCHEMA, { + organizations: [{ id: 'org_1', name: 'Acme Corp', isPersonal: false }], + }); + expect(parsed.success).toBe(false); + }); +}); diff --git a/src/lib/response-schemas.ts b/src/lib/response-schemas.ts index de86431..c551048 100644 --- a/src/lib/response-schemas.ts +++ b/src/lib/response-schemas.ts @@ -36,6 +36,7 @@ import type { BatchRerunResponse, BatchRunFreshResponse, ListRunsResponse, + RerunAdvisory, RerunClosure, RerunResponse, RunResponse, @@ -92,8 +93,13 @@ export const RUN_RESPONSE_SCHEMA: v.GenericSchema = v.loos createdAt: v.string(), startedAt: v.nullish(v.string(), null), finishedAt: v.nullish(v.string(), null), - codeVersion: v.string(), - targetUrl: v.string(), + // Both are nullable on the wire (`RunEnvelope` declares + // `[string, 'null']`): `codeVersion` is null on pre-M3.1 rows and on tests + // with no stored code body, `targetUrl` is null for backend runs and for + // execution backends that record no URL. Renderers already omit the line + // when either is null (rule 3). + codeVersion: v.nullish(v.string(), null), + targetUrl: v.nullish(v.string(), null), createdFrom: v.nullish(v.string(), null), failedStepIndex: v.nullish(v.number(), null), failureKind: v.nullish(v.string(), null), @@ -103,9 +109,25 @@ export const RUN_RESPONSE_SCHEMA: v.GenericSchema = v.loos videoUrl: v.nullish(v.string(), null), stepSummary: RUN_STEP_SUMMARY_SCHEMA, retryAfterSeconds: v.optional(v.number()), - // Client-synthesized Portal link (never sent by the server); tolerated so a - // future server echo cannot fail validation. - dashboardUrl: v.optional(v.string()), + // Portal link. Newer backends DO send this (they alone know which store + // answered the read and can resolve a non-prod portal origin); older ones + // omit it and the CLI computes its own. `nullish` rather than `optional` + // deliberately: the backend omits the field when no correct link exists, but + // a `null` from any other producer must not fail validation and take down + // `test wait` — the same trap that had to be un-sprung for a null + // `targetUrl`/`codeVersion`. + // + // The `undefined` default (NOT `null`, unlike every field above) is load-bearing + // and measured: valibot applies a default only when the key is absent, and + // skips the assignment entirely when that default is `undefined` — so an + // omitted field stays an ABSENT key, which is exactly what + // `withRunDashboardUrl`'s `'dashboardUrl' in run` test reads to decide + // "old backend, compute the link myself". Aligning this with the + // `nullish(..., null)` fields above would materialize the key on every + // response and silently kill that fallback. A wire `null` is preserved as + // null here (nullable passes it through untouched) and normalized at the + // consumer, not in the schema. Locked by tests in response-schemas.test.ts. + dashboardUrl: v.nullish(v.string(), undefined), // Absence means "steps not requested" and drives command branching, so no // default is applied (rule 3, optional branch). steps: v.optional(v.nullable(v.array(RUN_STEP_DTO_SCHEMA))), @@ -144,6 +166,18 @@ const RERUN_CLOSURE_SCHEMA: v.GenericSchema = v.looseObje clearedCaptured: v.number(), }); +/** + * Mirrors `RerunAdvisory` (runs.types.ts): a server-side note that a + * requested option was forwarded to the execution engine but is not yet + * honored there. Present only on a V3-routed rerun that explicitly opted + * out of auto-heal — absent everywhere else, so this schema is only ever + * used inside an `v.optional(v.array(...))` wrapper. + */ +const RERUN_ADVISORY_SCHEMA: v.GenericSchema = v.looseObject({ + feature: v.string(), + message: v.string(), +}); + /** Mirrors `RerunResponse` (runs.types.ts): `POST /tests/{testId}/runs/rerun`. */ export const RERUN_RESPONSE_SCHEMA: v.GenericSchema = v.looseObject({ runId: v.string(), @@ -154,6 +188,11 @@ export const RERUN_RESPONSE_SCHEMA: v.GenericSchema = v. // FE reruns omit `closure`; the CLI's `!!closure` truthy check relies on // absent staying absent, so optional with no default (rule 3). closure: v.optional(v.nullable(RERUN_CLOSURE_SCHEMA)), + // Absent on every response except a V3-routed rerun with an explicit + // autoHeal:false opt-out (rule 3: optional, no default, so presence/absence + // survives validation byte-identically). Older backends that predate the + // field simply omit it — never fails validation. + advisories: v.optional(v.array(RERUN_ADVISORY_SCHEMA)), }); // --------------------------------------------------------------------------- @@ -185,6 +224,10 @@ export const BATCH_RERUN_RESPONSE_SCHEMA: v.GenericSchema; + /** + * The calling key's own org binding (mirrors `CliOrgBinding`). Present + * only for a Postgres-backed membership key (`sk-member-…`); `name` is + * nullable (best-effort resolution). + */ + org?: { id: string; name: string | null; role: string }; } +/** Mirrors `CliOrgSummary` (lib/org-render.ts): one `Me.organizations[]` entry. */ +const ORG_SUMMARY_SCHEMA = v.looseObject({ + id: v.string(), + name: v.string(), + role: v.string(), + isPersonal: v.boolean(), +}); + +/** Mirrors `CliOrgBinding` (lib/org-render.ts): `Me.org`. */ +const ORG_BINDING_SCHEMA = v.looseObject({ + id: v.string(), + name: v.nullable(v.string()), + role: v.string(), +}); + /** Mirrors `MeIdentity` (commands/doctor.ts): `GET /api/cli/v1/me` core. */ export const ME_IDENTITY_SCHEMA: v.GenericSchema = v.looseObject({ userId: v.optional(v.string()), keyId: v.optional(v.string()), + organizations: v.optional(v.array(ORG_SUMMARY_SCHEMA)), + org: v.optional(ORG_BINDING_SCHEMA), }); diff --git a/src/lib/runs.types.ts b/src/lib/runs.types.ts index 15501c9..2ec5666 100644 --- a/src/lib/runs.types.ts +++ b/src/lib/runs.types.ts @@ -47,6 +47,20 @@ export interface RerunClosure { clearedCaptured: number; } +/** + * A machine-readable note that a requested option was forwarded to the + * execution engine but is not yet honored there. Emitted ONLY on a + * V3-routed rerun when the caller explicitly requested `autoHeal:false` — + * absent otherwise, including every V2 response and every V3 response that + * did not request an opt-out. + */ +export interface RerunAdvisory { + /** Machine-readable feature name the advisory concerns (e.g. "autoHeal"). */ + feature: string; + /** Human-readable explanation of the current limitation. */ + message: string; +} + /** * Response from `POST /api/cli/v1/tests/{testId}/runs/rerun`. * FE shape: no `closure`. BE shape: includes `closure` with per-member runIds. @@ -67,6 +81,12 @@ export interface RerunResponse { * the `!!closure` truthy check in the CLI already handles both). */ closure?: RerunClosure | null; + /** + * Present only when this rerun was routed to V3 execution AND the caller + * explicitly requested `autoHeal:false` (see {@link RerunAdvisory}). Absent + * on every other response, including older backends that predate the field. + */ + advisories?: RerunAdvisory[]; } /** @@ -122,6 +142,10 @@ export interface BatchRerunClosure { * Present when the server supports partial-accept (SOME ids bad → 200 with accepted+notFound). * When ALL ids are bad the server returns 404 (caught separately in the catch block). * Optional for back-compat with older backends that don't send this field. + * `advisories` = present only when at least one FE test in the batch was routed to V3 + * execution AND explicitly requested `autoHeal:false` (see {@link RerunAdvisory}). + * Absent on every other response. The CLI aggregates + dedupes this field across + * chunked dispatch requests (see `dedupeRerunAdvisories` in `commands/test.ts`). */ export interface BatchRerunResponse { accepted: BatchRerunAccepted[]; @@ -129,6 +153,7 @@ export interface BatchRerunResponse { conflicts: BatchRerunConflict[]; closure: BatchRerunClosure; notFound?: string[]; + advisories?: RerunAdvisory[]; } /** @@ -195,8 +220,19 @@ export interface RunResponse { createdAt: string; startedAt: string | null; finishedAt: string | null; - codeVersion: string; - targetUrl: string; + /** + * Code version stamped at trigger time. `null` on pre-M3.1 rows and on + * runs whose test has no stored code body (e.g. a plan-driven frontend + * test), per the server contract (`RunEnvelope.codeVersion` is + * `[string, 'null']`). + */ + codeVersion: string | null; + /** + * Target URL stamped at trigger time. `null` for backend runs and for any + * run whose execution backend does not record one, per the server contract + * (`RunEnvelope.targetUrl` is `[string, 'null']`). + */ + targetUrl: string | null; createdFrom: string | null; failedStepIndex: number | null; failureKind: string | null; @@ -207,12 +243,21 @@ export interface RunResponse { /** Optional hint from the server; honored by the polling loop. */ retryAfterSeconds?: number; /** - * CLIENT-synthesized Portal deep link — never sent by the server. The CLI - * adds it to terminal run output (`test run --wait`, `test wait`, - * `test rerun --wait`) when projectId+testId are present and the API - * endpoint maps to a known portal host (see `resolvePortalUrl`). + * Portal deep link for this run's test. + * + * **Server-provided when present, client-synthesized otherwise.** Newer + * backends send this field on `GET /runs/{runId}` because only the server + * knows which store answered the read (a V3-served run's page is a different + * route family, and the V2 route it replaces cannot render for a V3-native + * project) and only the server can resolve the portal origin outside prod. + * + * A server `null` means "no correct link exists for this run" — the CLI + * treats it as absent for rendering and does NOT substitute its own guess. + * A missing field (older backend) reopens the client path: `resolvePortalUrl` + * from projectId+testId when the endpoint maps to a known portal host. See + * `withRunDashboardUrl` in `commands/test.ts`. */ - dashboardUrl?: string; + dashboardUrl?: string | null; /** * Full ordered step list. Only present when the request includes * `?includeSteps=true`. Absent (undefined) when the flag was not sent. @@ -290,7 +335,8 @@ export interface RunHistoryItem { /** May be null — backend doesn't always stamp it. */ startedAt: string | null; finishedAt: string | null; - codeVersion: string; + /** May be null — not stamped on pre-M3.2 rows or on tests with no code body. */ + codeVersion: string | null; /** Null when the run passed. */ failureKind: string | null; /** diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index 2b26c0e..98dd258 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -4,6 +4,7 @@ import type { OutputMode } from './output.js'; import { SKILL_NUDGE_COMMANDS, SKILL_NUDGE_OPT_OUT_ENV, + isPlanTemplateInvocation, isVerifySkillInstalled, maybeEmitSkillNudge, type SkillNudgeContext, @@ -207,3 +208,32 @@ describe('maybeEmitSkillNudge', () => { expect(probed.every(p => toPosix(p).startsWith('/work/here'))).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// isPlanTemplateInvocation — src/index.ts's preAction hook uses +// this to exempt `test create --plan-template` from BOTH the skill nudge +// above and the update-registry check in update-check.ts. Extracted here +// (rather than left inline in src/index.ts, which executes `program.parse()` +// at import time and so cannot safely be imported by a unit test) purely so +// the boolean logic is directly unit-testable. +// --------------------------------------------------------------------------- + +describe('isPlanTemplateInvocation', () => { + it('true for `test create` with planTemplate: true', () => { + expect(isPlanTemplateInvocation('test create', true)).toBe(true); + }); + + it('false for `test create` without planTemplate (undefined)', () => { + expect(isPlanTemplateInvocation('test create', undefined)).toBe(false); + }); + + it('false for `test create` with planTemplate: false', () => { + expect(isPlanTemplateInvocation('test create', false)).toBe(false); + }); + + it('false for any other command path even with planTemplate: true (Commander would never actually set this, but the check must not false-positive)', () => { + expect(isPlanTemplateInvocation('test create-batch', true)).toBe(false); + expect(isPlanTemplateInvocation('test run', true)).toBe(false); + expect(isPlanTemplateInvocation('auth status', true)).toBe(false); + }); +}); diff --git a/src/lib/skill-nudge.ts b/src/lib/skill-nudge.ts index 0f67572..4aa4d28 100644 --- a/src/lib/skill-nudge.ts +++ b/src/lib/skill-nudge.ts @@ -35,6 +35,29 @@ export const SKILL_NUDGE_COMMANDS: ReadonlySet = new Set([ */ export const SKILL_NUDGE_OPT_OUT_ENV = 'TESTSPRITE_NO_SKILL_WARNING'; +/** + * True when this invocation is `test create --plan-template`, a + * pure-local/informational flag (prints the plan-file skeleton and exits) + * that must be treated like `setup` / `agent install`: exempt from BOTH the + * missing-skill nudge above AND the update-registry check + * (`src/lib/update-check.ts`'s `maybeNotifyUpdate`, which hits the network + * and writes `~/.testsprite/update-check.json` — both contradict "no + * network" for this flag). Neither allowlist (`SKILL_NUDGE_COMMANDS` here, + * the unconditional call site in `src/index.ts`) tracks individual flags, + * only whole commands, so `src/index.ts`'s `preAction` hook filters this one + * case via this pure, independently-testable helper instead of teaching + * either module about flags. Exported from `skill-nudge.ts` (rather than + * `src/index.ts`, which executes `program.parse()` at import time and so + * cannot be safely imported by a unit test) purely so it has a home that + * supports direct unit testing. + */ +export function isPlanTemplateInvocation( + commandPath: string, + planTemplate: boolean | undefined, +): boolean { + return commandPath === 'test create' && planTemplate === true; +} + export interface SkillPresenceDeps { existsSync?: (p: string) => boolean; readFileSync?: (p: string) => string; diff --git a/src/lib/telemetry.spec.ts b/src/lib/telemetry.spec.ts new file mode 100644 index 0000000..1a53692 --- /dev/null +++ b/src/lib/telemetry.spec.ts @@ -0,0 +1,272 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { CommanderError } from 'commander'; +import { describe, expect, it, vi } from 'vitest'; +import { ApiError, CLIError, InterruptError, RequestTimeoutError } from './errors.js'; +import { + buildTelemetryEvent, + classifyCliError, + isTelemetryOptedOut, + recordOutcome, + resolveTelemetryAuth, +} from './telemetry.js'; + +function writeCreds(withKey: boolean): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-telemetry-')); + const p = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + const body = withKey + ? `[default]\napi_url = https://api.example.com\napi_key = sk-user-test\n` + : `[default]\napi_url = https://api.example.com\n`; + writeFileSync(p, body, { mode: 0o600 }); + return p; +} + +function okResponse(): Response { + return new Response(null, { status: 204 }); +} + +// --------------------------------------------------------------------------- +// classifyCliError — mirrors index.ts's exit-code mapping +// --------------------------------------------------------------------------- + +describe('classifyCliError', () => { + it('InterruptError → abort + INTERRUPTED + its exit code', () => { + expect(classifyCliError(new InterruptError('SIGINT'))).toEqual({ + outcome: 'abort', + exitCode: 130, + errorCode: 'INTERRUPTED', + }); + }); + + it('RequestTimeoutError → error + REQUEST_TIMEOUT + exit 7', () => { + expect(classifyCliError(new RequestTimeoutError(1000))).toEqual({ + outcome: 'error', + exitCode: 7, + errorCode: 'REQUEST_TIMEOUT', + }); + }); + + it('ApiError → error + its code + its exit code', () => { + const err = ApiError.authRequired(); + expect(classifyCliError(err)).toEqual({ + outcome: 'error', + exitCode: err.exitCode, + errorCode: err.code, + }); + }); + + it('CommanderError help/version → success + exit 0 (no errorCode)', () => { + expect(classifyCliError(new CommanderError(0, 'commander.helpDisplayed', ''))).toEqual({ + outcome: 'success', + exitCode: 0, + }); + expect(classifyCliError(new CommanderError(0, 'commander.version', ''))).toEqual({ + outcome: 'success', + exitCode: 0, + }); + }); + + it('CommanderError parse error → error + VALIDATION_ERROR + exit 5', () => { + expect(classifyCliError(new CommanderError(1, 'commander.unknownCommand', 'nope'))).toEqual({ + outcome: 'error', + exitCode: 5, + errorCode: 'VALIDATION_ERROR', + }); + }); + + it('CLIError → error + its exit code (no errorCode)', () => { + expect(classifyCliError(new CLIError('boom', 4))).toEqual({ outcome: 'error', exitCode: 4 }); + }); + + it('unknown error → error + exit 1', () => { + expect(classifyCliError(new Error('weird'))).toEqual({ outcome: 'error', exitCode: 1 }); + }); +}); + +// --------------------------------------------------------------------------- +// isTelemetryOptedOut +// --------------------------------------------------------------------------- + +describe('isTelemetryOptedOut', () => { + it('opts out on TESTSPRITE_NO_TELEMETRY or DO_NOT_TRACK truthy values', () => { + expect(isTelemetryOptedOut({ TESTSPRITE_NO_TELEMETRY: '1' })).toBe(true); + expect(isTelemetryOptedOut({ DO_NOT_TRACK: '1' })).toBe(true); + expect(isTelemetryOptedOut({ DO_NOT_TRACK: 'true' })).toBe(true); + }); + + it('does NOT opt out for unset / "0" / "false" / empty', () => { + expect(isTelemetryOptedOut({})).toBe(false); + expect(isTelemetryOptedOut({ DO_NOT_TRACK: '0' })).toBe(false); + expect(isTelemetryOptedOut({ TESTSPRITE_NO_TELEMETRY: 'false' })).toBe(false); + expect(isTelemetryOptedOut({ TESTSPRITE_NO_TELEMETRY: '' })).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// buildTelemetryEvent — allowlist only, no PII +// --------------------------------------------------------------------------- + +describe('buildTelemetryEvent', () => { + it('emits only allowlisted fields; never a url or message', () => { + const event = buildTelemetryEvent( + { + command: 'test run', + outcome: 'error', + exitCode: 5, + errorCode: 'VALIDATION_ERROR', + durationMs: 42, + output: 'json', + endpointUrl: 'https://secret.internal', + profile: 'work', + }, + { CI: '1' }, + false, + ); + expect(event).toEqual({ + command: 'test run', + outcome: 'error', + exitCode: 5, + errorCode: 'VALIDATION_ERROR', + durationMs: 42, + cliVersion: expect.any(String), + os: process.platform, + nodeVersion: process.versions.node, + output: 'json', + ci: true, + }); + // Allowlist backstop: nothing sensitive leaked through. + expect(event).not.toHaveProperty('endpointUrl'); + expect(event).not.toHaveProperty('profile'); + expect(event).not.toHaveProperty('message'); + }); + + it('ci=true when non-TTY even without CI env; omits errorCode when absent', () => { + const event = buildTelemetryEvent( + { command: 'test list', outcome: 'success', exitCode: 0, durationMs: 1 }, + {}, + false, + ); + expect(event.ci).toBe(true); + expect(event).not.toHaveProperty('errorCode'); + }); +}); + +// --------------------------------------------------------------------------- +// recordOutcome — gates + POST shape + best-effort +// --------------------------------------------------------------------------- + +describe('recordOutcome', () => { + const base = { command: 'test run', outcome: 'success' as const, exitCode: 0, durationMs: 42 }; + + it('POSTs the event to the beacon when authenticated', async () => { + const fetchImpl = vi.fn().mockResolvedValue(okResponse()); + await recordOutcome( + { ...base, output: 'json' }, + { env: {}, credentialsPath: writeCreds(true), fetchImpl, isTTY: true }, + ); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + const [url, init] = fetchImpl.mock.calls[0] as [string, RequestInit]; + expect(url).toMatch(/\/api\/cli\/v1\/telemetry$/); + const headers = init.headers as Record; + expect(headers['x-api-key']).toBe('sk-user-test'); + const body = JSON.parse(init.body as string) as Record; + expect(body.command).toBe('test run'); + expect(body.outcome).toBe('success'); + expect(body.ci).toBe(false); + expect(body).not.toHaveProperty('endpointUrl'); + }); + + it('skips when opted out (DO_NOT_TRACK)', async () => { + const fetchImpl = vi.fn(); + await recordOutcome(base, { + env: { DO_NOT_TRACK: '1' }, + credentialsPath: writeCreds(true), + fetchImpl, + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('skips under --dry-run', async () => { + const fetchImpl = vi.fn(); + await recordOutcome( + { ...base, dryRun: true }, + { env: {}, credentialsPath: writeCreds(true), fetchImpl }, + ); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('skips on the abort path (Ctrl-C stays snappy)', async () => { + const fetchImpl = vi.fn(); + await recordOutcome( + { ...base, outcome: 'abort', exitCode: 130, errorCode: 'INTERRUPTED' }, + { env: {}, credentialsPath: writeCreds(true), fetchImpl }, + ); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('skips when no leaf command ran', async () => { + const fetchImpl = vi.fn(); + await recordOutcome( + { ...base, command: '' }, + { env: {}, credentialsPath: writeCreds(true), fetchImpl }, + ); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('skips when no api-key is configured (authenticated-only)', async () => { + const fetchImpl = vi.fn(); + await recordOutcome(base, { env: {}, credentialsPath: writeCreds(false), fetchImpl }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('is best-effort — a fetch rejection never propagates', async () => { + const fetchImpl = vi.fn().mockRejectedValue(new Error('network down')); + await expect( + recordOutcome(base, { env: {}, credentialsPath: writeCreds(true), fetchImpl }), + ).resolves.toBeUndefined(); + }); + + it('uses pre-resolved auth and never reads the credentials file', async () => { + const fetchImpl = vi.fn().mockResolvedValue(okResponse()); + await recordOutcome(base, { + env: {}, + // Bogus path: if it were read, no key would resolve and the POST would skip. + credentialsPath: join(tmpdir(), 'cli-telemetry-missing', 'credentials'), + resolvedAuth: { apiKey: 'sk-user-pre', apiUrl: 'https://api.example.com' }, + fetchImpl, + }); + expect(fetchImpl).toHaveBeenCalledTimes(1); + const [, init] = fetchImpl.mock.calls[0] as [string, RequestInit]; + expect((init.headers as Record)['x-api-key']).toBe('sk-user-pre'); + }); + + it('skips when the pre-resolved auth carries no api-key', async () => { + const fetchImpl = vi.fn(); + await recordOutcome(base, { + env: {}, + resolvedAuth: { apiUrl: 'https://api.example.com' }, + fetchImpl, + }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); + +// --------------------------------------------------------------------------- +// resolveTelemetryAuth — one config read, done up front for `auth remove` +// --------------------------------------------------------------------------- + +describe('resolveTelemetryAuth', () => { + it('returns the api-key and api-url from the credentials file', () => { + const auth = resolveTelemetryAuth({}, { env: {}, credentialsPath: writeCreds(true) }); + expect(auth).toEqual({ apiKey: 'sk-user-test', apiUrl: 'https://api.example.com' }); + }); + + it('returns no api-key when the profile has none', () => { + const auth = resolveTelemetryAuth({}, { env: {}, credentialsPath: writeCreds(false) }); + expect(auth.apiKey).toBeUndefined(); + expect(auth.apiUrl).toBe('https://api.example.com'); + }); +}); diff --git a/src/lib/telemetry.ts b/src/lib/telemetry.ts new file mode 100644 index 0000000..6fe5c40 --- /dev/null +++ b/src/lib/telemetry.ts @@ -0,0 +1,240 @@ +/** + * Client-side telemetry: one "command outcome" event per CLI invocation, + * fired to the backend beacon (`POST /api/cli/v1/telemetry`), which forwards it + * to PostHog server-side. The CLI ships no PostHog SDK / write-key. + * + * Contract: + * - Best-effort: every failure is swallowed; telemetry NEVER changes a + * command's behavior, output, or exit code. + * - Bounded: the POST is aborted after {@link TELEMETRY_TIMEOUT_MS}, so the + * flush-before-exit can add at most that long to a slow/unreachable-backend + * invocation (normal case is a sub-200ms POST). + * - Authenticated-only: skipped when no api-key is configured (the backend + * keys the event on the api-key's user; anonymous events are out of scope). + * - Opt-out: skipped when `TESTSPRITE_NO_TELEMETRY` or the cross-tool + * `DO_NOT_TRACK` is set. Also skipped under `--dry-run` (no network) and + * when no leaf command actually ran (bare `--help` / parse errors). + * - Never on abort (SIGINT/SIGTERM): Ctrl-C must exit immediately, so the + * abort path emits nothing (awaiting would delay shutdown; fire-and-forget + * would be cut off before delivery anyway). + * + * Privacy: the event is a fixed allowlist of low-cardinality, PII-free fields. + * It NEVER carries target URLs, api keys, flag values, arg values, or error + * messages — `errorCode` is a stable machine code, never the human message. + */ +import { CommanderError } from 'commander'; +import { loadConfig } from './config.js'; +import { defaultCredentialsPath } from './credentials.js'; +import { ApiError, CLIError, InterruptError, RequestTimeoutError } from './errors.js'; +import { facadeBaseUrl } from './facade.js'; +import { VERSION } from '../version.js'; + +/** + * Max time the flush-before-exit will wait on the beacon POST. Kept short so a + * slow/unreachable backend adds at most this to a command's exit (including + * Ctrl-C); a healthy POST completes in well under it, and when it doesn't, + * dropping the event is the correct best-effort behavior. + */ +export const TELEMETRY_TIMEOUT_MS = 1000; + +export type TelemetryOutcome = 'success' | 'error' | 'abort'; + +export interface TelemetryOutcomeInput { + /** Leaf command path that ran, e.g. `test run`. Empty → skip (no command). */ + command: string; + outcome: TelemetryOutcome; + exitCode: number; + /** Stable machine error code (e.g. `VALIDATION_ERROR`) — never a message. */ + errorCode?: string; + durationMs: number; + /** Global flags, used to resolve config + fill context. */ + profile?: string; + endpointUrl?: string; + output?: string; + dryRun?: boolean; +} + +export interface TelemetryDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: typeof globalThis.fetch; + /** Test seam for TTY detection (CI/non-interactive context prop). */ + isTTY?: boolean; + /** + * Auth pre-resolved by the caller (the preAction hook). When present, + * recordOutcome uses it verbatim instead of re-reading the credentials file — + * so a command that mutates that file (`auth remove` deletes the profile) is + * still reported on the key it ran under. + */ + resolvedAuth?: ResolvedTelemetryAuth; +} + +/** The (apiKey, apiUrl) pair telemetry needs, resolved once per invocation. */ +export interface ResolvedTelemetryAuth { + apiKey?: string; + apiUrl: string; +} + +/** The exact wire body — a flat allowlist mirroring the backend DTO. */ +export interface TelemetryEvent { + command: string; + outcome: TelemetryOutcome; + exitCode?: number; + errorCode?: string; + durationMs?: number; + cliVersion?: string; + os?: string; + nodeVersion?: string; + output?: string; + ci?: boolean; +} + +/** + * Map a thrown CLI error to its telemetry disposition — mirrors the exit-code + * mapping in `index.ts`'s top-level catch so telemetry and the process exit + * code never disagree. Pure; safe to unit-test. + */ +export function classifyCliError(err: unknown): { + outcome: TelemetryOutcome; + exitCode: number; + errorCode?: string; +} { + if (err instanceof InterruptError) { + return { outcome: 'abort', exitCode: err.exitCode, errorCode: 'INTERRUPTED' }; + } + if (err instanceof RequestTimeoutError) { + return { outcome: 'error', exitCode: err.exitCode, errorCode: 'REQUEST_TIMEOUT' }; + } + if (err instanceof ApiError) { + return { outcome: 'error', exitCode: err.exitCode, errorCode: err.code }; + } + if (err instanceof CommanderError) { + // Help / version are user-requested successes (exit 0); everything else + // Commander throws is a parse/validation error (exit 5). + if ( + err.code === 'commander.helpDisplayed' || + err.code === 'commander.help' || + err.code === 'commander.version' + ) { + return { outcome: 'success', exitCode: 0 }; + } + return { outcome: 'error', exitCode: 5, errorCode: 'VALIDATION_ERROR' }; + } + if (err instanceof CLIError) { + return { outcome: 'error', exitCode: err.exitCode }; + } + return { outcome: 'error', exitCode: 1 }; +} + +/** True when the operator has opted out via either supported env var. */ +export function isTelemetryOptedOut(env: NodeJS.ProcessEnv): boolean { + return isTruthyEnv(env.TESTSPRITE_NO_TELEMETRY) || isTruthyEnv(env.DO_NOT_TRACK); +} + +function isTruthyEnv(v: string | undefined): boolean { + if (v === undefined) return false; + const t = v.trim().toLowerCase(); + return t !== '' && t !== '0' && t !== 'false'; +} + +/** Assemble the allowlisted wire event. No URL / message / flag value ever. */ +export function buildTelemetryEvent( + input: TelemetryOutcomeInput, + env: NodeJS.ProcessEnv, + isTTY: boolean, +): TelemetryEvent { + return { + command: input.command, + outcome: input.outcome, + exitCode: input.exitCode, + ...(input.errorCode ? { errorCode: input.errorCode } : {}), + durationMs: input.durationMs, + cliVersion: VERSION, + os: process.platform, + nodeVersion: process.versions.node, + ...(input.output === 'json' || input.output === 'text' ? { output: input.output } : {}), + ci: isTruthyEnv(env.CI) || !isTTY, + }; +} + +/** + * Resolve just the (apiKey, apiUrl) pair telemetry needs. Called once from the + * `index.ts` preAction hook — BEFORE the command's own action runs — so a + * command that mutates the credentials file (`auth remove` deletes the profile) + * is still reported on the key it ran under. The hook only calls this for a + * telemetry-eligible, non-opted-out, non-dry-run invocation, so a gated-out or + * opted-out call never reads the credentials file. + */ +export function resolveTelemetryAuth( + opts: { profile?: string; endpointUrl?: string }, + deps: { env?: NodeJS.ProcessEnv; credentialsPath?: string } = {}, +): ResolvedTelemetryAuth { + const config = loadConfig({ + profile: opts.profile ?? 'default', + endpointUrl: opts.endpointUrl, + env: deps.env ?? process.env, + credentialsPath: deps.credentialsPath ?? defaultCredentialsPath(), + }); + return { apiKey: config.apiKey, apiUrl: config.apiUrl }; +} + +/** + * Fire one command-outcome event to the beacon. Awaited by `index.ts` before + * `process.exit` (the flush) — bounded and fully best-effort, so it can neither + * hang nor throw. Skips silently when opted out, under dry-run, with no leaf + * command, or when no api-key is configured. + */ +export async function recordOutcome( + input: TelemetryOutcomeInput, + deps: TelemetryDeps = {}, +): Promise { + try { + const env = deps.env ?? process.env; + if (isTelemetryOptedOut(env)) return; + if (input.dryRun) return; + if (!input.command) return; // no leaf command ran (bare --help / parse error) + // Never on the abort path: Ctrl-C / SIGTERM must exit immediately. Awaiting + // a beacon post here would delay shutdown by up to the bounded timeout, and + // a fire-and-forget post would be cut off by process.exit before delivery — + // so the event would be unreliable anyway. Aborts are simply not reported. + if (input.outcome === 'abort') return; + + // Prefer auth the preAction hook already resolved (before a command like + // `auth remove` could delete the profile); fall back to a fresh read. + const config = + deps.resolvedAuth ?? + loadConfig({ + profile: input.profile ?? 'default', + endpointUrl: input.endpointUrl, + env, + credentialsPath: deps.credentialsPath ?? defaultCredentialsPath(), + }); + if (!config.apiKey) return; // authenticated-only + + const url = `${facadeBaseUrl(config.apiUrl)}/telemetry`; + const isTTY = deps.isTTY ?? process.stderr.isTTY === true; + const body = buildTelemetryEvent(input, env, isTTY); + const fetchImpl = deps.fetchImpl ?? globalThis.fetch; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), TELEMETRY_TIMEOUT_MS); + // Don't keep the event loop alive just for the timer. + if (typeof timer.unref === 'function') timer.unref(); + try { + await fetchImpl(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-api-key': config.apiKey, + 'user-agent': `testsprite-cli/${VERSION}`, + }, + body: JSON.stringify(body), + signal: controller.signal, + }); + } finally { + clearTimeout(timer); + } + } catch { + // Best-effort: telemetry must never affect the command. + } +} diff --git a/src/lib/v3-advisory.test.ts b/src/lib/v3-advisory.test.ts index 124720f..05eac19 100644 --- a/src/lib/v3-advisory.test.ts +++ b/src/lib/v3-advisory.test.ts @@ -9,11 +9,17 @@ describe('routingLabel', () => { }); describe('V3 routing advisory', () => { - it('names each open behavior gap (cancel, delete, target-url)', () => { + it('names the open behavior gaps', () => { const text = V3_ROUTING_ADVISORY.join('\n'); - expect(text).toContain('test cancel'); - expect(text).toContain('test delete'); expect(text).toContain('--target-url'); + expect(text).toContain('rerun'); + }); + + // Warning about behavior that now works trains users to ignore the block. + it('no longer warns about gaps that have shipped', () => { + const text = V3_ROUTING_ADVISORY.join('\n'); + expect(text).not.toContain('test cancel'); + expect(text).not.toContain('zombie'); }); it('emitV3RoutingAdvisory writes every line to the sink', () => { diff --git a/src/lib/v3-advisory.ts b/src/lib/v3-advisory.ts index a8b75ed..262c467 100644 --- a/src/lib/v3-advisory.ts +++ b/src/lib/v3-advisory.ts @@ -11,12 +11,20 @@ export function routingLabel(v3Enabled: boolean): 'v3' | 'v2' { return v3Enabled ? 'v3' : 'v2'; } -/** Consolidated advisory (stderr) emitted when V3 routing is on. */ +/** + * Consolidated advisory (stderr) emitted when V3 routing is on. + * + * Only genuinely-open gaps belong here. Two originally-listed items have + * shipped and were removed: `test cancel` works on V3 runs, and `test delete` + * mirrors into Postgres instead of leaving a runnable, billable row behind. + * An advisory that warns about fixed behavior is worse than none — it teaches + * users to distrust the whole block, and it sends them chasing a failure that + * cannot happen. + */ export const V3_ROUTING_ADVISORY: string[] = [ '[advisory] V3 routing is on for this account. While these gaps are open:', - ' - `test cancel` may return 404', - ' - `test delete` may leave a zombie run', - ' - `--target-url` is ignored on frontend runs', + ' - `--target-url` is ignored on frontend runs (the run uses the project environment)', + ' - a frontend rerun replays the run it was pointed at, not necessarily the latest saved code', ]; /** Write the advisory to a stderr sink, one line per call. */ diff --git a/src/version.ts b/src/version.ts index 5b301a6..80229a2 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.4.0'; +export const VERSION = '0.5.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index bdee849..1e4802c 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -7,30 +7,37 @@ Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Kiro, Windsurf, Copilot, Codex) Options: - -h, --help display help for command + -h, --help display help for command Commands: - install [options] Write the TestSprite agent skills (verification loop + - first-run onboarding) into a project for a coding agent - list List supported agent targets and skills, their status, and - landing paths - status [options] Check installed TestSprite skill files against this CLI - version: ok, stale, modified, unmarked, absent, or corrupt - (exits 1 when anything needs attention, so it can gate CI) - help [command] display help for command + install [options] [targets...] Write the TestSprite agent skills + (verification loop + first-run onboarding) + into a project for a coding agent. Target(s) + may be given positionally (e.g. \`agent + install cursor codex\`) and/or via --target; + the two are merged. + list List supported agent targets and skills, + their status, and landing paths + status [options] Check installed TestSprite skill files + against this CLI version: ok, stale, + modified, unmarked, absent, or corrupt (exits + 1 when anything needs attention, so it can + gate CI) + help [command] display help for command " `; exports[`--help snapshots > agent install 1`] = ` -"Usage: testsprite agent install [options] +"Usage: testsprite agent install [options] [targets...] Write the TestSprite agent skills (verification loop + first-run onboarding) -into a project for a coding agent +into a project for a coding agent. Target(s) may be given positionally (e.g. +\`agent install cursor codex\`) and/or via --target; the two are merged. Options: --target Agent target(s): claude, cursor, cline, antigravity, kiro, - windsurf, copilot, codex (comma-separated or repeated) - (default: []) + windsurf, copilot, codex (comma-separated or repeated). + Merged with any positional target(s). (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -78,9 +85,22 @@ exports[`--help snapshots > auth configure 1`] = ` "Usage: testsprite auth configure [options] Options: - --from-env Read TESTSPRITE_API_KEY (and optionally TESTSPRITE_API_URL) from - the environment instead of prompting (default: false) - -h, --help display help for command + --api-key API key to configure (skips the interactive prompt) + --from-env Read TESTSPRITE_API_KEY from the environment instead of + prompting (default: false) + --agent Coding-agent target to install: claude, antigravity, + cursor, cline, kiro, windsurf, copilot, codex (default: + claude) (default: "claude") + --no-agent Skip the agent skill install (configure credentials + only) + --force Overwrite an existing skill file (a .bak backup is + kept) + --dir Project root for the skill install (default: current + directory) + -y, --yes Non-interactive: accept all defaults, never prompt + --skip-if-configured Skip the API key prompt when credentials already exist + for this profile (CI-safe idempotent re-run) + -h, --help display help for command " `; @@ -302,12 +322,17 @@ Commands: is recorded as error: in its row and folded into exit 7) 7 timeout or per-member poll error — resume with: testsprite test wait 10 transport/network failure (UNAVAILABLE) — retry the command + 11 rate limited, and nothing else went wrong — polls are retried + automatically honoring Retry-After first, so this means the throttle + outlasted that budget while the runs were still fine. Back off, then + re-attach with test wait. (A throttle that instead consumes the whole + --timeout reports 7, and any real timeout or failure keeps 7 / 1.) On failure/blocked/cancelled, run: testsprite test artifact get Ctrl-C detaches only (the run keeps executing and billing); stop it for real with: testsprite test cancel - rerun [options] [test-ids...] Re-execute a test (or multiple) as a cheap replay — FE replays the saved script (no credit), BE re-runs the dependency closure. + rerun [options] [test-ids...] Re-execute a test (or multiple) as a replay — FE replays the saved script, BE re-runs the dependency closure. Billed the same as a fresh run: 0.5 credits per FE rerun / 0.2 credits per BE rerun (legacy V2 accounts: FE rerun remains free). Exit codes: 0 passed (or queued without --wait) @@ -397,6 +422,86 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeou " `; +exports[`--help snapshots > test create 1`] = ` +"Usage: testsprite test create [options] + +Create a test from saved code (--code-file) or an agent-supplied plan +(--plan-from, FE-only, M3.2 piece-5) + +Options: + --project project id (returned by \`testsprite project list\`) + --type frontend|backend + --name human-readable test name (becomes \`title\` in + storage) + --description optional human description (≤ 2000 chars) + --priority optional priority — one of: p0, p1, p2, p3 + --code-file file containing the test code (≤ 350 KB) + --plan-from JSON file with the full FE test definition — + projectId, type, name, planSteps[] all live in the + file (≤ 256 KB; mutually exclusive with + --code-file). In this mode + --project/--type/--name/--description/--priority + are ignored. + --plan-template print a minimal valid plan-file skeleton to stdout + and exit (pure-local: no network, no credentials, + ignores every other flag). Pipe to a file and + edit: \`--plan-template > plan.json\`. (default: + false) + --run after create, trigger the test. Combine with + --wait to block until terminal. (default: false) + --wait with --run, poll until terminal status (default: + false) + --timeout with --run --wait, max seconds to wait + --target-url with --run, override the project default env URL + --idempotency-key opaque idempotency token (1-256 ASCII chars). + Defaults to a UUIDv4 minted per invocation; pin + one yourself for safe retries. + --produces BE only: variable name this test captures + (repeatable). Drives dependency-aware wave + ordering on \`test rerun\` and \`test run --all\`. + (default: []) + --needs BE only: variable name this test consumes + (repeatable). Use to declare upstream producer + dependencies. (default: []) + --category BE only: test category. Use 'teardown' or + 'cleanup' to mark a final-wave cleanup test. + -h, --help display help for command + +Plan file format (--plan-from ) — minimal valid example: + +{ + "$schema": "https://raw.githubusercontent.com/TestSprite/testsprite-cli/v0.5.0/schemas/plan.schema.json", + "projectId": "prj_abc123", + "type": "frontend", + "name": "Login rejects an empty password", + "planSteps": [ + { + "type": "action", + "description": "Navigate to /login and submit the form with an empty password" + }, + { + "type": "assertion", + "description": "Verify an inline error says the password is required" + } + ] +} + +Print this exact skeleton: testsprite test create --plan-template +Validate offline (no network): testsprite test create --plan-from --dry-run +Multiple tests: test create-batch --plans | --plan-from-dir +Full field reference: DOCUMENTATION.md -> "Plan file format" + + +BE dependency authoring (M4): + --produces/--needs drive wave ordering on \`test rerun\` + \`test run --all\`. + --category teardown marks a final-wave cleanup test. + These flags are backend-only; supplying with --type frontend is an error (exit 5). + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test create-batch 1`] = ` "Usage: testsprite test create-batch [options] @@ -471,8 +576,11 @@ Options: -h, --help display help for command Notes: - • Frontend replays are free verbatim script replays (no credit); backend replays - re-run the dependency closure and may cost credits — a one-line advisory is printed. + • Each replay is billed as a rerun — 0.5 credits for a frontend replay (verbatim + script), 0.2 credits for a backend replay (re-runs the dependency closure) — + same price as a fresh run, so \`--runs N\` costs roughly N×0.5 credits for a + frontend test (legacy V2 accounts: FE rerun remains free). A one-line advisory + is printed before a backend replay. • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass; this measures replay stability of the saved script against the configured URL. • \`--output json\` emits a machine-readable stability report for CI gating. @@ -526,7 +634,7 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeou exports[`--help snapshots > test rerun 1`] = ` "Usage: testsprite test rerun [options] [test-ids...] -Re-execute a test (or multiple) as a cheap replay — FE replays the saved script (no credit), BE re-runs the dependency closure. +Re-execute a test (or multiple) as a replay — FE replays the saved script, BE re-runs the dependency closure. Billed the same as a fresh run: 0.5 credits per FE rerun / 0.2 credits per BE rerun (legacy V2 accounts: FE rerun remains free). Exit codes: 0 passed (or queued without --wait) @@ -587,6 +695,8 @@ Notes: slow trigger/poll under load is not cut at the 120s default (see --request-timeout). • Batch --wait: rate-deferred tests appear in \`deferred[]\` and \`summary.deferred\`, and force a non-zero exit — they are NOT counted in \`summary.total\` (dispatched only). + • On V3-routed accounts, --no-auto-heal is still rolling out and may not yet be + honored server-side — check \`auth status\` for your routing. Dry-run shape notes: • --dry-run shows the BE rerun wire shape (includes \`closure{}\`); FE rerun responses @@ -623,6 +733,8 @@ Options: --page-size with --history: number of runs per page (1–100, default 20) --cursor with --history: opaque cursor from a prior page + --rerun with --history: show only reruns + --no-rerun with --history: show only fresh (non-rerun) runs --columns with --history: select/reorder text table columns --no-header with --history: suppress the text table header row -h, --help display help for command diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 3b04712..a37ea5d 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -15,7 +15,7 @@ import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { execNpm } from './helpers/execNpm.js'; +import { assertFreshBuild } from './helpers/assertFreshBuild.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, '..'); @@ -33,12 +33,12 @@ let baseUrl: string; let tmpHome: string; beforeAll(async () => { - // Always rebuild — `npm run build` is fast and a stale `dist/index.js` - // would silently mask ESM/import regressions in this suite. The - // existsSync skip we used to do here let `dist` rot under - // refactors and gave false-green on `project list` once - // already. - execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + // The build now happens exactly once in `test/global-setup.ts`, before any + // test file runs — no per-file rebuild here, so this suite can + // never race a concurrent rebuild for access to `dist/`. Fail fast (missing + // OR stale relative to src/, e.g. watch-mode reruns) instead of silently + // spawning a binary that isn't the code under test. + assertFreshBuild(REPO_ROOT, BIN_PATH); server = createServer((req: IncomingMessage, res: ServerResponse) => { const url = req.url ?? '/'; if (url.startsWith('/api/cli/v1/projects/')) { @@ -179,6 +179,31 @@ beforeAll(async () => { ); return; } + if (testId === 'test_ambiguous_org' && subPath === undefined) { + res.writeHead(409, { 'content-type': 'application/json' }); + res.end( + JSON.stringify({ + error: { + code: 'CONFLICT', + message: + 'Test id "test_ambiguous_org" resolves in more than one of your organizations.', + nextAction: + 'This id is ambiguous across your organizations. Open the specific project in the ' + + 'Portal to act on it, or contact support — uuid-derived ids should not normally collide.', + requestId: 'req_subproc_ambiguous', + details: { + reason: 'ambiguous_org', + testId: 'test_ambiguous_org', + candidates: [ + { projectId: 'project_a', orgId: 'org_a' }, + { projectId: 'project_b', orgId: 'org_b' }, + ], + }, + }, + }), + ); + return; + } if (testId === 'test_subproc' && subPath === 'result') { res.writeHead(200, { 'content-type': 'application/json' }); res.end( @@ -386,7 +411,7 @@ function runCli(args: string[], envOverrides: Record = {}): Prom describe('auth status subprocess (+ deprecated whoami alias)', () => { it('prints JSON me and exits 0 against the local server', async () => { const result = await runCli(['auth', 'status', '--output', 'json'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -418,7 +443,7 @@ describe('auth status subprocess (+ deprecated whoami alias)', () => { it('text mode renders userId/scopes legibly', async () => { const result = await runCli(['auth', 'status'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -428,19 +453,19 @@ describe('auth status subprocess (+ deprecated whoami alias)', () => { it('--debug emits structured debug events to stderr without leaking the key', async () => { const result = await runCli(['--debug', 'auth', 'status', '--output', 'json'], { - TESTSPRITE_API_KEY: 'sk-subproc-secret', + TESTSPRITE_API_KEY: 'sk-user-subproc-secret', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); expect(result.stderr).toContain('"kind":"request"'); expect(result.stderr).toContain('"kind":"response"'); - expect(result.stderr).not.toContain('sk-subproc-secret'); + expect(result.stderr).not.toContain('sk-user-subproc-secret'); expect(result.stderr).not.toContain('x-api-key'); }, 30_000); it('deprecated `auth whoami` alias still works and prints a deprecation notice', async () => { const result = await runCli(['auth', 'whoami', '--output', 'json'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -453,7 +478,7 @@ describe('auth status subprocess (+ deprecated whoami alias)', () => { describe('project list subprocess', () => { it('--output json returns the §6.1 ProjectList shape', async () => { const result = await runCli(['--output', 'json', 'project', 'list'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -465,7 +490,7 @@ describe('project list subprocess', () => { it('text output renders a header row and the project name', async () => { const result = await runCli(['project', 'list'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -484,7 +509,7 @@ describe('project list subprocess', () => { it('--page-size 0 exits 5 (VALIDATION_ERROR), not 1 (generic)', async () => { const result = await runCli(['--output', 'json', 'project', 'list', '--page-size', '0'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(5); @@ -496,7 +521,7 @@ describe('project list subprocess', () => { // Previously silently clamped to 100; now rejected at exit 5 so callers // get fast feedback that the value is out of range (Fix 7 — B-E2E-01 wave). const result = await runCli(['--output', 'json', 'project', 'list', '--page-size', '101'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(5); @@ -512,7 +537,7 @@ describe('project list subprocess', () => { const result = await runCli( ['--output', 'json', '--request-timeout', '30s', 'project', 'list'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }, ); @@ -533,7 +558,7 @@ describe('malformed --endpoint-url is rejected (exit 5), not retried as a networ it('an unparseable endpoint exits 5 with a VALIDATION_ERROR naming endpoint-url', async () => { const result = await runCli( ['--output', 'json', '--endpoint-url', 'not a url', 'project', 'list'], - { TESTSPRITE_API_KEY: 'sk-subproc' }, + { TESTSPRITE_API_KEY: 'sk-user-subproc' }, ); expect(result.exitCode).toBe(5); const parsed = JSON.parse(result.stderr) as { error: { code: string; nextAction: string } }; @@ -544,7 +569,7 @@ describe('malformed --endpoint-url is rejected (exit 5), not retried as a networ it('a non-http(s) scheme exits 5 instead of being retried as a network failure', async () => { const result = await runCli( ['--output', 'json', '--endpoint-url', 'ftp://example.com', 'project', 'list'], - { TESTSPRITE_API_KEY: 'sk-subproc' }, + { TESTSPRITE_API_KEY: 'sk-user-subproc' }, ); expect(result.exitCode).toBe(5); const parsed = JSON.parse(result.stderr) as { error: { code: string } }; @@ -571,7 +596,7 @@ describe('invalid --output is rejected uniformly (exit 5)', () => { it('auth status --output yaml exits 5 before any network call', async () => { const result = await runCli(['--output', 'yaml', 'auth', 'status'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(5); @@ -586,7 +611,7 @@ describe('a malformed --profile is rejected (exit 5), not silently corrupting cr // any credential read/write path. it('exits 5 with a VALIDATION_ERROR naming the profile flag', async () => { const result = await runCli(['--output', 'json', '--profile', 'prod]', 'project', 'list'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(5); @@ -599,7 +624,7 @@ describe('a malformed --profile is rejected (exit 5), not silently corrupting cr describe('project get subprocess', () => { it('--output json returns the §6.1 Project shape', async () => { const result = await runCli(['--output', 'json', 'project', 'get', 'project_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -611,7 +636,7 @@ describe('project get subprocess', () => { it('text output prints the labeled fields block', async () => { const result = await runCli(['project', 'get', 'project_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -621,7 +646,7 @@ describe('project get subprocess', () => { it('exits 4 (NOT_FOUND) for an unknown project id', async () => { const result = await runCli(['--output', 'json', 'project', 'get', 'project_does_not_exist'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(4); @@ -635,7 +660,7 @@ describe('test list subprocess', () => { const result = await runCli( ['--output', 'json', 'test', 'list', '--project', 'project_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }, ); @@ -651,7 +676,7 @@ describe('test list subprocess', () => { const result = await runCli( ['--output', 'json', 'test', 'list', '--project', 'project_subproc', '--type', 'frontend'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }, ); @@ -664,7 +689,7 @@ describe('test list subprocess', () => { const result = await runCli( ['--output', 'json', 'test', 'list', '--project', 'project_subproc', '--type', 'backend'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }, ); @@ -674,7 +699,7 @@ describe('test list subprocess', () => { it('text output renders header + status column', async () => { const result = await runCli(['test', 'list', '--project', 'project_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -695,7 +720,7 @@ describe('test list subprocess', () => { it('missing --project exits 5 with VALIDATION_ERROR (typed envelope)', async () => { const result = await runCli(['--output', 'json', 'test', 'list'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); // Per the CLI error spec §2 ("missing required field" → VALIDATION_ERROR) @@ -712,7 +737,7 @@ describe('test list subprocess', () => { it('--type=junk exits 5 with VALIDATION_ERROR (local validation)', async () => { const result = await runCli( ['--output', 'json', 'test', 'list', '--project', 'project_subproc', '--type', 'junk'], - { TESTSPRITE_API_KEY: 'sk-subproc', TESTSPRITE_API_URL: baseUrl }, + { TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl }, ); expect(result.exitCode).toBe(5); const parsed = JSON.parse(result.stderr) as { error: { code: string } }; @@ -723,7 +748,7 @@ describe('test list subprocess', () => { describe('test get subprocess', () => { it('--output json returns the §6.2 Test shape', async () => { const result = await runCli(['--output', 'json', 'test', 'get', 'test_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -735,7 +760,7 @@ describe('test get subprocess', () => { it('text output prints the labeled fields block', async () => { const result = await runCli(['test', 'get', 'test_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -746,7 +771,7 @@ describe('test get subprocess', () => { it('exits 4 (NOT_FOUND) for an unknown test id', async () => { const result = await runCli(['--output', 'json', 'test', 'get', 'test_does_not_exist'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(4); @@ -755,10 +780,44 @@ describe('test get subprocess', () => { }, 30_000); }); +describe('AMBIGUOUS_ORG conflict rendering (test get)', () => { + it('text mode: exit 6, prints one candidate line per colliding project + a --project hint', async () => { + const result = await runCli(['test', 'get', 'test_ambiguous_org'], { + TESTSPRITE_API_KEY: 'sk-user-subproc', + TESTSPRITE_API_URL: baseUrl, + }); + expect(result.exitCode).toBe(6); + expect(result.stderr).toContain('resolves in more than one of your organizations'); + expect(result.stderr).toContain('candidate: project project_a (org org_a)'); + expect(result.stderr).toContain('candidate: project project_b (org org_b)'); + expect(result.stderr).toContain('--project '); + }, 30_000); + + it('--output json: exit 6, error envelope carries code + candidates verbatim', async () => { + const result = await runCli(['--output', 'json', 'test', 'get', 'test_ambiguous_org'], { + TESTSPRITE_API_KEY: 'sk-user-subproc', + TESTSPRITE_API_URL: baseUrl, + }); + expect(result.exitCode).toBe(6); + const parsed = JSON.parse(result.stderr) as { + error: { + code: string; + details: { reason: string; candidates: Array<{ projectId: string; orgId: string }> }; + }; + }; + expect(parsed.error.code).toBe('AMBIGUOUS_ORG'); + expect(parsed.error.details.reason).toBe('ambiguous_org'); + expect(parsed.error.details.candidates).toEqual([ + { projectId: 'project_a', orgId: 'org_a' }, + { projectId: 'project_b', orgId: 'org_b' }, + ]); + }, 30_000); +}); + describe('test code get subprocess', () => { it('--output json returns the §6.3 TestCode shape', async () => { const result = await runCli(['--output', 'json', 'test', 'code', 'get', 'test_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -777,7 +836,7 @@ describe('test code get subprocess', () => { it('text mode prints the inline source body without a JSON envelope', async () => { const result = await runCli(['test', 'code', 'get', 'test_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -791,7 +850,7 @@ describe('test code get subprocess', () => { it('exits 4 (NOT_FOUND) for an unknown test id', async () => { const result = await runCli( ['--output', 'json', 'test', 'code', 'get', 'test_does_not_exist'], - { TESTSPRITE_API_KEY: 'sk-subproc', TESTSPRITE_API_URL: baseUrl }, + { TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl }, ); expect(result.exitCode).toBe(4); const parsed = JSON.parse(result.stderr) as { error: { code: string } }; @@ -802,7 +861,7 @@ describe('test code get subprocess', () => { describe('test steps subprocess', () => { it('--output json returns the §6.4 TestStepList shape', async () => { const result = await runCli(['--output', 'json', 'test', 'steps', 'test_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -821,7 +880,7 @@ describe('test steps subprocess', () => { it('text mode renders the step table and shared run metadata', async () => { const result = await runCli(['test', 'steps', 'test_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -837,7 +896,7 @@ describe('test steps subprocess', () => { describe('test result subprocess', () => { it('--output json returns the §6.5 LatestResult shape with correlation block', async () => { const result = await runCli(['--output', 'json', 'test', 'result', 'test_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -861,7 +920,7 @@ describe('test result subprocess', () => { it('text mode highlights failureKind + failedStepIndex above timestamps', async () => { const result = await runCli(['test', 'result', 'test_subproc'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -881,7 +940,7 @@ describe('auth remove subprocess', () => { it('removes the profile file entry and exits 0', async () => { // First configure a profile (via the consolidated `setup` path) const configureResult = await runCli(['setup', '--from-env', '--no-agent'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(configureResult.exitCode).toBe(0); @@ -895,7 +954,7 @@ describe('auth remove subprocess', () => { describe('setup --from-env subprocess', () => { it('writes the credentials file with mode 0600', async () => { const result = await runCli(['setup', '--from-env', '--no-agent'], { - TESTSPRITE_API_KEY: 'sk-mode-test', + TESTSPRITE_API_KEY: 'sk-user-mode-test', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(0); @@ -1070,7 +1129,7 @@ describe('--dry-run subprocess smoke', () => { it('auth logout --dry-run does NOT delete credentials', async () => { // First configure a real profile so there's something to (not) delete. await runCli(['setup', '--from-env', '--no-agent'], { - TESTSPRITE_API_KEY: 'sk-keep-me', + TESTSPRITE_API_KEY: 'sk-user-keep-me', TESTSPRITE_API_URL: baseUrl, }); const credPath = join(tmpHome, '.testsprite', 'credentials'); @@ -1201,7 +1260,7 @@ describe('test artifact get --dry-run subprocess (Item-9 regression)', () => { describe('[fix-5] Commander parse errors → exit 5; help/version → exit 0', () => { it('`test result` with no test-id argument exits 5', async () => { const result = await runCli(['test', 'result'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); // Missing required argument is a VALIDATION_ERROR family error → exit 5. @@ -1234,7 +1293,7 @@ describe('[fix-5] Commander parse errors → exit 5; help/version → exit 0', ( // a coding agent parsing stderr does not receive an unexpected plain-text // error and crash its JSON.parse. const result = await runCli(['--output', 'json', 'test', 'result'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(5); @@ -1251,7 +1310,7 @@ describe('[fix-5] Commander parse errors → exit 5; help/version → exit 0', ( // --output json appears before the unknown subcommand, so Commander parses // it and program.opts().output is 'json' when the error fires. const result = await runCli(['--output', 'json', 'test', 'not-a-real-subcommand'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(5); @@ -1266,7 +1325,7 @@ describe('[fix-5] Commander parse errors → exit 5; help/version → exit 0', ( // The error fires before --output json is parsed, so program.opts().output // is the default 'text'. The argv fallback scan must still detect json mode. const result = await runCli(['test', 'not-a-real-subcommand', '--output', 'json'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(5); @@ -1279,7 +1338,7 @@ describe('[fix-5] Commander parse errors → exit 5; help/version → exit 0', ( it('text mode: Commander parse error still emits plain text (no regression)', async () => { const result = await runCli(['test', 'result'], { - TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_KEY: 'sk-user-subproc', TESTSPRITE_API_URL: baseUrl, }); expect(result.exitCode).toBe(5); diff --git a/test/contract/p4-schema.test.ts b/test/contract/p4-schema.test.ts index b255b57..ac2f5ae 100644 --- a/test/contract/p4-schema.test.ts +++ b/test/contract/p4-schema.test.ts @@ -277,7 +277,7 @@ function makeCreds(): { credentialsPath: string } { // sans the /api/cli/v1 suffix that facadeBaseUrl re-appends). writeFileSync( credentialsPath, - `[default]\napi_url = https://api.testsprite.com\napi_key = sk-test\n`, + `[default]\napi_url = https://api.testsprite.com\napi_key = sk-user-test\n`, { mode: 0o600 }, ); return { credentialsPath }; diff --git a/test/contract/p5-schema.test.ts b/test/contract/p5-schema.test.ts index b68ef56..6a7edb3 100644 --- a/test/contract/p5-schema.test.ts +++ b/test/contract/p5-schema.test.ts @@ -190,7 +190,7 @@ function makeCreds(): { credentialsPath: string } { mkdirSync(dir, { recursive: true }); writeFileSync( credentialsPath, - `[default]\napi_url = https://api.testsprite.com\napi_key = sk-test\n`, + `[default]\napi_url = https://api.testsprite.com\napi_key = sk-user-test\n`, { mode: 0o600 }, ); return { credentialsPath }; diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 4fb4584..822a5ef 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -865,3 +865,79 @@ describe('matrix coverage guard', () => { it.skip('bootstrap tip after auth configure — see auth.test.ts for tip coverage', () => { // No-op: piece-3 unit tests in src/commands/auth.test.ts cover the tip. }); + +// --------------------------------------------------------------------------- +// 13. Positional target argument +// +// Repro: `agent install ` (the exact one-liner form documented in +// DOCUMENTATION.md / README for all 8 targets) previously installed the +// claude skill regardless of the target named, because `install` declared +// only `--target ` with no positional `.argument()` — Commander silently +// dropped the excess positional and the non-TTY default-to-claude path won. +// These tests drive the real built binary (not just the command wiring) to +// pin the documented one-liner behavior for good. +// --------------------------------------------------------------------------- +describe('positional target argument', () => { + it('installs the named target, not the claude default (repro: agent install cursor)', () => { + const tmpDir = freshTmpDir(); + const result = runCli(['agent', 'install', 'cursor', '--dir', tmpDir, '--output', 'json']); + expect(result.status).toBe(0); + + expect(existsSync(join(tmpDir, pathFor('cursor', 'testsprite-verify')))).toBe(true); + expect(existsSync(join(tmpDir, pathFor('claude', 'testsprite-verify')))).toBe(false); + }); + + it('accepts every documented one-liner form (agent install ) for all 8 targets', () => { + for (const target of Object.keys(TARGETS) as AgentTarget[]) { + const tmpDir = freshTmpDir(); + const result = runCli(['agent', 'install', target, '--dir', tmpDir, '--output', 'json']); + expect(result.status, `exit code for positional '${target}'`).toBe(0); + expect( + existsSync(join(tmpDir, pathFor(target, 'testsprite-verify'))), + `landing file for positional '${target}'`, + ).toBe(true); + } + }); + + it('accepts multiple positional targets in one invocation', () => { + const tmpDir = freshTmpDir(); + const result = runCli([ + 'agent', + 'install', + 'cline', + 'kiro', + '--dir', + tmpDir, + '--output', + 'json', + ]); + expect(result.status).toBe(0); + expect(existsSync(join(tmpDir, pathFor('cline', 'testsprite-verify')))).toBe(true); + expect(existsSync(join(tmpDir, pathFor('kiro', 'testsprite-verify')))).toBe(true); + }); + + it('merges a positional target with --target', () => { + const tmpDir = freshTmpDir(); + const result = runCli([ + 'agent', + 'install', + 'antigravity', + '--target=windsurf', + '--dir', + tmpDir, + '--output', + 'json', + ]); + expect(result.status).toBe(0); + expect(existsSync(join(tmpDir, pathFor('antigravity', 'testsprite-verify')))).toBe(true); + expect(existsSync(join(tmpDir, pathFor('windsurf', 'testsprite-verify')))).toBe(true); + }); + + it('rejects an unknown positional target with exit 5 instead of silently defaulting', () => { + const tmpDir = freshTmpDir(); + const result = runCli(['agent', 'install', 'banana', '--dir', tmpDir]); + expect(result.status).toBe(5); + expect(result.stderr).toContain('unknown target "banana"'); + expect(existsSync(join(tmpDir, pathFor('claude', 'testsprite-verify')))).toBe(false); + }); +}); diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index 5a07434..2e755d0 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -95,7 +95,7 @@ describe('setup --dry-run --no-agent', () => { const credsTmpDir = freshTmpDir(); const result = runCli( - ['--dry-run', 'setup', '--api-key', 'sk-dry-no-agent', '--no-agent', '--dir', tmpDir], + ['--dry-run', 'setup', '--api-key', 'sk-user-dry-no-agent', '--no-agent', '--dir', tmpDir], { HOME: credsTmpDir }, ); @@ -124,7 +124,7 @@ describe('setup --dry-run (with agent)', () => { '--dry-run', 'setup', '--api-key', - 'sk-dry-with-agent', + 'sk-user-dry-with-agent', '--agent', 'claude', '--dir', @@ -198,7 +198,7 @@ describe('deprecated `init` alias', () => { const credsTmpDir = freshTmpDir(); const result = runCli( - ['--dry-run', 'init', '--api-key', 'sk-dep-init', '--no-agent', '--dir', tmpDir], + ['--dry-run', 'init', '--api-key', 'sk-user-dep-init', '--no-agent', '--dir', tmpDir], { HOME: credsTmpDir }, ); @@ -249,7 +249,7 @@ describe('setup --agent --no-agent conflict warn fires through real binary', '--dry-run', 'setup', '--api-key', - 'sk-conflict-e2e', + 'sk-user-conflict-e2e', '--agent', 'cursor', '--no-agent', diff --git a/test/e2e/signal.e2e.test.ts b/test/e2e/signal.e2e.test.ts index 6e03743..d91ea80 100644 --- a/test/e2e/signal.e2e.test.ts +++ b/test/e2e/signal.e2e.test.ts @@ -83,7 +83,7 @@ async function waitAndInterrupt( { env: { ...process.env, - TESTSPRITE_API_KEY: 'sk-e2e-signal', + TESTSPRITE_API_KEY: 'sk-user-e2e-signal', TESTSPRITE_API_URL: baseUrl, TESTSPRITE_NO_SKILL_WARNING: '1', TESTSPRITE_NO_UPDATE_NOTIFIER: '1', @@ -152,7 +152,7 @@ describe('signal e2e — graceful detach during test wait (DEV-331)', () => { const child = spawn(process.execPath, [BIN_PATH, 'test', 'list', '--project', 'p1'], { env: { ...process.env, - TESTSPRITE_API_KEY: 'sk-e2e-signal', + TESTSPRITE_API_KEY: 'sk-user-e2e-signal', TESTSPRITE_API_URL: baseUrl, TESTSPRITE_NO_SKILL_WARNING: '1', TESTSPRITE_NO_UPDATE_NOTIFIER: '1', diff --git a/test/e2e/skill-nudge.e2e.test.ts b/test/e2e/skill-nudge.e2e.test.ts index d0930bd..36f9e77 100644 --- a/test/e2e/skill-nudge.e2e.test.ts +++ b/test/e2e/skill-nudge.e2e.test.ts @@ -52,7 +52,7 @@ function homeWithCreds(): string { mkdirSync(join(home, '.testsprite'), { recursive: true }); writeFileSync( join(home, '.testsprite', 'credentials'), - `[default]\napi_key = sk-fake-nudge\napi_url = ${DEAD_ENDPOINT}\n`, + `[default]\napi_key = sk-user-fake-nudge\napi_url = ${DEAD_ENDPOINT}\n`, 'utf8', ); return home; @@ -137,4 +137,18 @@ describe('skill nudge — suppression gates', () => { }, NETWORK_TIMEOUT_MS, ); + + // `test create --plan-template` is pure-local and informational + // (same family as `setup` / `agent install`) — it must not trip the + // "test create" entry in SKILL_NUDGE_COMMANDS even with a configured + // profile and no skill installed. Regression guard for the flag-aware + // skip added to src/index.ts's preAction hook. + it('is silent for `test create --plan-template` even with a configured profile and no skill installed', () => { + const proj = freshDir('ts-nudge-proj-'); + const home = homeWithCreds(); + const result = runCli(['test', 'create', '--plan-template'], { cwd: proj, home }); + expect(result.status).toBe(0); + expect(result.stderr).not.toContain(WARN_SUBSTR); + expect(result.stdout).toContain('"planSteps"'); + }); }); diff --git a/test/global-setup.ts b/test/global-setup.ts new file mode 100644 index 0000000..1c8531f --- /dev/null +++ b/test/global-setup.ts @@ -0,0 +1,31 @@ +/** + * Vitest `globalSetup` for the main unit suite (`npm test` / `npm run + * test:coverage`). + * + * Builds the CLI exactly once, synchronously, in the main process BEFORE + * any test file is collected or run. + * + * `test/cli.subprocess.test.ts` and `test/help.snapshot.test.ts` both spawn + * the built `dist/index.js` as a real child process, and used to rebuild it + * themselves inside their own `beforeAll`. On a cold or contended `dist/` + * (notably the public `release.yaml` gate, where `test:coverage` runs + * BEFORE the explicit `build` step) two independent in-suite rebuilds could + * overlap with a concurrent spawn of the binary they were still writing, + * producing a flaky non-zero exit unrelated to the assertion under test. + * + * `globalSetup` runs once, before any worker spawns — building here instead + * guarantees a single, complete build finishes before Vitest ever imports + * or spawns anything, eliminating the race at the root rather than papering + * over it with `fileParallelism: false` alone (kept for other hermeticity + * reasons, but no longer load-bearing for this specific flake). + */ +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execNpm } from './helpers/execNpm.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, '..'); + +export default function setup(): void { + execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); +} diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index b6c23b4..beb5251 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -5,15 +5,17 @@ * * Lives under `test/` * (not `src/`) to mirror the existing subprocess test pattern — the - * snapshot runs the real built binary and therefore needs a build in - * `beforeAll`, the same way `test/cli.subprocess.test.ts` does. + * snapshot runs the real built binary. The build itself happens exactly + * once in `test/global-setup.ts`, before any test file runs, so + * this suite only asserts the binary is there rather than rebuilding it + * itself. */ import { execFileSync } from 'node:child_process'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { beforeAll, describe, expect, it } from 'vitest'; -import { execNpm } from './helpers/execNpm.js'; +import { assertFreshBuild } from './helpers/assertFreshBuild.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, '..'); @@ -42,6 +44,8 @@ const cases: Array<[string, string[]]> = [ ['test rerun', ['test', 'rerun', '--help']], ['test flaky', ['test', 'flaky', '--help']], // R5: regression guard for commands that gained new flag wording + // Locks the "Plan file format" example + --plan-template pointer + ['test create', ['test', 'create', '--help']], ['test create-batch', ['test', 'create-batch', '--help']], ['test run', ['test', 'run', '--help']], // DEV-331 piece 3 @@ -50,7 +54,9 @@ const cases: Array<[string, string[]]> = [ describe('--help snapshots', () => { beforeAll(() => { - execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + // Missing OR stale (watch-mode reruns skip globalSetup's build) → fail + // fast instead of snapshotting a binary that isn't the code under test. + assertFreshBuild(REPO_ROOT, BIN_PATH); }); for (const [name, args] of cases) { diff --git a/test/helpers/assertFreshBuild.ts b/test/helpers/assertFreshBuild.ts new file mode 100644 index 0000000..4ea5504 --- /dev/null +++ b/test/helpers/assertFreshBuild.ts @@ -0,0 +1,42 @@ +import { existsSync, readdirSync, statSync } from 'node:fs'; +import { join } from 'node:path'; + +/** + * Fail-fast invariant for suites that spawn the built `dist/index.js`. + * + * `test/global-setup.ts` builds exactly once per `vitest` INVOCATION — + * which is correct for `npm test` / CI, but under `npm run test:watch` + * a source edit re-runs the suites WITHOUT re-running globalSetup, so a + * subprocess suite would silently exercise a stale binary (false green / + * false red). Rebuilding here would reintroduce the build race that + * `globalSetup` exists to avoid, so instead this check is read-only: if + * anything under `src/` is newer than the built entrypoint, throw with + * instructions rather than let a stale binary masquerade as the code + * under test. + */ +export function assertFreshBuild(repoRoot: string, binPath: string): void { + if (!existsSync(binPath)) { + throw new Error( + `Built CLI not found at ${binPath}. Expected test/global-setup.ts to build it before this suite runs.`, + ); + } + const binMtime = statSync(binPath).mtimeMs; + const newest = newestMtimeUnder(join(repoRoot, 'src')); + if (newest > binMtime) { + throw new Error( + `dist/ is stale: a file under src/ is newer than ${binPath}. ` + + `Vitest watch mode re-runs suites without re-running globalSetup's build — ` + + `run \`npm run build\` (or restart \`npm test\`) so this suite spawns the code under test.`, + ); + } +} + +function newestMtimeUnder(dir: string): number { + let newest = 0; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + const mtime = entry.isDirectory() ? newestMtimeUnder(path) : statSync(path).mtimeMs; + if (mtime > newest) newest = mtime; + } + return newest; +} diff --git a/test/helpers/stdoutPurity.ts b/test/helpers/stdoutPurity.ts index f617cef..bf0336b 100644 --- a/test/helpers/stdoutPurity.ts +++ b/test/helpers/stdoutPurity.ts @@ -46,6 +46,7 @@ export async function expectJsonModeStdoutIsPureJson( throw new Error( `JSON-mode stdout is not parseable JSON: ${(err as Error).message}\n` + `--- stdout (${trimmed.length} bytes) ---\n${trimmed.slice(0, 400)}\n--- end ---`, + { cause: err }, ); } diff --git a/test/mock-backend/handlers.smoke.test.ts b/test/mock-backend/handlers.smoke.test.ts index 04c553f..7604be7 100644 --- a/test/mock-backend/handlers.smoke.test.ts +++ b/test/mock-backend/handlers.smoke.test.ts @@ -35,7 +35,13 @@ import { mockBackend.installLifecycle(); -const VALID_KEY = 'tsp_dev_canary_key'; +// Deliberately NOT credential-shaped. This is a mock-backend header value, but +// `tsp_dev_…` matched the release LEAK_RE's membership-namespace pattern once +// that pattern learned `tsp_` — a false positive in the one scan whose whole +// job is to refuse to publish a real key. Renaming is better than adding an +// exception: an exception for `tsp_dev_*` would also mask a real credential +// that happened to start that way. +const VALID_KEY = 'mock-backend-accepts-any-key'; function get(path: string, init: RequestInit = {}): Promise { const headers = new Headers(init.headers); diff --git a/vitest.config.ts b/vitest.config.ts index bd9b2ad..04f62ab 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -7,8 +7,12 @@ export default defineConfig({ // Strip real TESTSPRITE_* env vars and redirect the home dir so results // never depend on the developer's shell or ~/.testsprite (see the file). setupFiles: ['./test/helpers/hermetic-env.ts'], - // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel - // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). + // Build the CLI exactly once, before any test file/worker spawns, so the + // subprocess/snapshot suites never race an in-suite rebuild against a + // concurrent spawn of the binary they're still writing. + globalSetup: ['./test/global-setup.ts'], + // Kept as defense-in-depth: forces test files to run one at a time in a + // single worker, so no other per-file hook can race dist/ either. fileParallelism: false, coverage: { provider: 'v8', From aa517e7c0f7c58747e90b94b36da6b6481c03c8a Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Wed, 12 Aug 2026 23:48:32 +0000 Subject: [PATCH 112/117] release: v0.6.0 Private-Snapshot-RevId: a1d7d06fdc820c61fef1087b57c93a72145d39dd --- .github/workflows/security.yml | 244 ++++ .gitleaks.toml | 15 + CHANGELOG.md | 13 + DOCUMENTATION.md | 8 +- README.md | 1 + eslint.security.config.mjs | 50 + package-lock.json | 47 +- package.json | 3 +- scripts/README.md | 15 +- src/commands/auth.test.ts | 21 + src/commands/doctor.test.ts | 24 + src/commands/test.rerun.spec.ts | 194 +++- src/commands/test.run.spec.ts | 473 ++++++++ src/commands/test.test.ts | 1034 +++++++++++++++++ src/commands/test.ts | 586 ++++++++-- src/commands/usage.test.ts | 24 + src/lib/gh-output.test.ts | 136 +++ src/lib/gh-output.ts | 141 ++- src/lib/http.ts | 81 +- src/lib/poll.spec.ts | 13 +- src/lib/poll.ts | 132 ++- src/lib/response-schemas.ts | 36 +- src/lib/v3-advisory.test.ts | 24 +- src/lib/v3-advisory.ts | 26 + src/version.ts | 2 +- test/__snapshots__/help.snapshot.test.ts.snap | 24 +- test/cli.subprocess.test.ts | 5 + test/contract/p4-schema.test.ts | 20 +- test/contract/p5-schema.test.ts | 20 +- test/helpers/tempDir.ts | 31 + 30 files changed, 3252 insertions(+), 191 deletions(-) create mode 100644 .github/workflows/security.yml create mode 100644 eslint.security.config.mjs create mode 100644 test/helpers/tempDir.ts diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..55b36f6 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,244 @@ +# Ported from community PR https://github.com/TestSprite/testsprite-cli/pull/220 +# (author @OkeyAmy), adapted to this repo's actual constraints: +# +# - No CodeQL job. This repo (testsprite-cli-atlas) is a private Team-plan +# repo without GitHub Advanced Security purchased — the CodeQL Action +# (init/analyze) hard-fails here (`Code Security must be enabled for this +# repository to use code scanning`, verified against the live API before +# writing this file). CodeQL is free on the public mirror +# (TestSprite/testsprite-cli) instead, enabled there via GitHub's +# repository-level "default setup" — a SETTING, not a workflow file — so +# there is nothing here for the outbound snapshot to carry or exclude, +# and no risk of the two CodeQL configuration modes (default setup vs. +# workflow-based) conflicting on the public repo. +# - `dependency-review-action` has the same Advanced-Security requirement on +# private repos, so its job only executes once this file is running in +# the public mirror (see the job's own `if:`) — it still exists as a +# no-op step on every atlas PR/push rather than failing one. +# - `eslint-plugin-security` lints only files changed in this PR/push (see +# the `lint-security` job), not the whole tree: a full-tree run surfaces +# ~380 pre-existing findings (almost entirely +# `security/detect-non-literal-fs-filename` on ordinary local-path fs +# calls a config-file-reading CLI makes routinely — not a real +# vulnerability signal for this codebase shape). Blocking on that +# backlog on day one would make the job noise from the first run; this +# keeps every rule at its designed severity and holds new/changed code to +# it without silently exempting the existing tree from the rule. +# - Secret scanning: this repo's `ci.yml` already runs a gitleaks +# WORKING-TREE scan on every PR/push (added by atlas #274, 2026-07). This +# file adds the complementary FULL-HISTORY scan from #220 as its own job, +# scoped to `push` only (not every PR iteration) — full-history scanning +# needs `fetch-depth: 0` and walks every past commit, which is too slow +# to run per-PR (the same tradeoff `docs/internal/cli-oss/ +# supply-chain-hardening.md` §1.4 already made for `ci.yml`'s job) but is +# the only mode that catches a secret that was committed and later +# removed from the working tree. Uses the checksum-verified pinned-binary +# install pattern already established by `ci.yml`/`divergence-sentinel.yml` +# rather than `gitleaks/gitleaks-action` (#220's choice): that action +# requires a `GITLEAKS_LICENSE` for org-owned repos — this org owns both +# testsprite-cli repos, so "free mode" does not apply — and adding it +# would introduce a new third-party `uses:` entry that the public repo's +# `allowed_actions` allowlist does not carry. +# - No `pnpm-lock.yaml` (this is an npm-only repo — see package-lock.json) +# and no `sandbox/**` ignore additions (no such directory exists in this +# repo; #220 never explained the addition). + +name: Security + +on: + pull_request: + types: [opened, synchronize, reopened] + push: + branches: [main, dev, stg] + +# Least-privilege default (same convention as ci.yml). No job below needs +# write; anything more must opt in locally. +permissions: + contents: read + +jobs: + # ── 1. Dependency audit ──────────────────────────────────────────────────── + # Blocks on HIGH/CRITICAL in production dependencies (what ships to users). + # Dev-only vulns are reported but do not fail the build — they never reach + # a user's machine. + audit: + name: Dependency Audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + + - name: Audit production dependencies (blocking) + run: npm audit --omit=dev --audit-level=high + + - name: Audit all dependencies (informational) + run: npm audit --audit-level=high || true + + # ── 2. Dependency review on PRs (public mirror only) ────────────────────── + # Blocks PRs that introduce a new vulnerable package. Requires GitHub + # Advanced Security, which atlas (private) does not have — see header. + # This is exactly the point in the pipeline where a community PR against + # a new dependency actually lands, so gating to the public repo costs + # nothing. + dependency-review: + name: Dependency Review + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' && github.repository == 'TestSprite/testsprite-cli' + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4.9.0 + with: + fail-on-severity: high + + # ── 3. ESLint with security rules (changed files only) ──────────────────── + # See header for why this is diff-scoped rather than full-tree. + lint-security: + name: ESLint Security (changed files) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + + - name: Determine changed TypeScript files under src/ + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + set -euo pipefail + : > "$RUNNER_TEMP/changed-ts-files.txt" + rm -f "$RUNNER_TEMP/lint-empty-reason.txt" + # Two different things can leave that list empty, and they must not + # read the same in the log. "A real diff against a real base found + # no src TypeScript changes" is normal and stays green. "We had no + # base to diff against, so we never looked" must NOT be a green + # no-op — a green check that inspected nothing is worse than no + # check, because it reads as coverage. So this step never just + # skips: it either resolves a real base to diff (possibly via a + # fallback) or it fails the job outright. + # + # A pull_request always supplies a base SHA and fetch-depth: 0 makes + # it resolvable, so an unusable base there means this job's own + # checkout/base resolution is broken — fail loudly instead of + # reporting a false green. A push can legitimately have no usable + # base (github.event.before is all-zeros on a first push to a + # branch, or unreachable after certain force-pushes); on push we + # fall back to a real diff against HEAD^ (the tip commit's own + # diff, reachable thanks to fetch-depth: 0) instead of skipping, and + # only fall back further — to linting the full tracked src/**/*.ts + # set — when HEAD itself has no parent (a genuine first commit). + if [ -n "${BASE_SHA:-}" ] && git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then + RESOLVED_BASE="$BASE_SHA" + BASE_DESC="$BASE_SHA (event-provided base)" + elif [ "$EVENT_NAME" = "pull_request" ]; then + echo "::error title=Security lint broken::pull_request base SHA ('${BASE_SHA:-}') is empty or unreachable even with fetch-depth: 0. A pull_request always supplies a resolvable base, so this means base resolution is broken, not that there is nothing to check. Failing the job instead of reporting a false green." >&2 + exit 1 + elif git rev-parse --verify -q HEAD^ >/dev/null 2>&1; then + RESOLVED_BASE="HEAD^" + BASE_DESC="HEAD^ (fallback: push event supplied no usable base SHA — was '${BASE_SHA:-}' — so diffing HEAD's own commit instead of skipping)" + else + RESOLVED_BASE="" + BASE_DESC="none (fallback: HEAD has no parent, a genuine first commit; linting the full tracked src/**/*.ts set instead of skipping)" + fi + + echo "Base used: ${BASE_DESC}" + + if [ -n "$RESOLVED_BASE" ]; then + git diff --name-only --diff-filter=ACMR -z "$RESOLVED_BASE" "$HEAD_SHA" -- ':(glob)src/**/*.ts' \ + > "$RUNNER_TEMP/changed-ts-files.txt" + else + git ls-files -z -- ':(glob)src/**/*.ts' > "$RUNNER_TEMP/changed-ts-files.txt" + fi + + # NUL-terminated, so a newline count (`wc -l`) would be wrong here — + # count NUL bytes instead (git filenames can never contain one, so + # this always equals the number of entries). + FILE_COUNT=$(tr -dc '\0' < "$RUNNER_TEMP/changed-ts-files.txt" | wc -c) + echo "Files matched: ${FILE_COUNT}" + + if [ "$FILE_COUNT" -eq 0 ]; then + echo "::notice title=Security lint: genuinely nothing to check::${BASE_DESC} produced zero src/**/*.ts files. This is a real result (empty diff, or an empty tree), not a skip." + echo "EMPTY" > "$RUNNER_TEMP/lint-empty-reason.txt" + fi + + - name: Run security lint on changed files + run: | + set -euo pipefail + # The file list is NUL-separated end-to-end: git diff/ls-files -z -> + # file -> xargs -0. Git filenames may contain a literal newline (or + # start with `-`), so converting to newlines (as this step used to, + # via `tr '\0' '\n'` + `xargs -d '\n'`) lets a filename like + # "src/x.ts\n--no-error-on-unmatched-pattern\nnot-a-real-file.ts" + # split into a real path, an injected ESLint OPTION, and a + # nonexistent path — ESLint then exits 0 having never linted the + # malicious file. NUL is the one byte git filenames cannot contain, + # so `-0`/`-z` is the only splitting mode that is actually safe + # here (the previous comment on this step claiming xargs was doing + # "NUL/newline-safe" splitting was false: the `tr` upstream had + # already destroyed the NUL delimiters before xargs ever saw the + # list). The trailing `--` additionally stops ESLint's own option + # parser at the file-list boundary, so no filename starting with + # `-` can be misread as a flag. + if [ -f "$RUNNER_TEMP/lint-empty-reason.txt" ]; then + echo "Zero files to lint (see the previous step's log for which base was used and why) — nothing to lint, reporting success on a genuine empty result, not a skip." + exit 0 + fi + FILE_COUNT=$(tr -dc '\0' < "$RUNNER_TEMP/changed-ts-files.txt" | wc -c) + echo "Linting ${FILE_COUNT} file(s)." + xargs -a "$RUNNER_TEMP/changed-ts-files.txt" -0 \ + npx eslint --config eslint.security.config.mjs --format stylish -- + + # ── 4. Full-history secret scan (push only) ──────────────────────────────── + # Complements ci.yml's working-tree gitleaks job (every PR/push) with the + # one thing that job structurally cannot see: a secret that was committed + # and later removed. See header for why this doesn't also run per-PR. + gitleaks-history: + name: Secret scan — full history (gitleaks) + runs-on: ubuntu-latest + if: github.event_name == 'push' + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + fetch-depth: 0 # full history required for a history-mode scan + + # Same pinned version + checksum-verified install as ci.yml / + # divergence-sentinel.yml — see those files for why this pin. + - name: Install gitleaks + env: + GITLEAKS_VERSION: '8.28.0' + run: | + set -euo pipefail + BASE="https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}" + TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -sSL -o "$TARBALL" "${BASE}/${TARBALL}" + curl -sSL -o checksums.txt "${BASE}/gitleaks_${GITLEAKS_VERSION}_checksums.txt" + grep " ${TARBALL}\$" checksums.txt | sha256sum -c - + tar -xzf "$TARBALL" gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Scan full git history for secrets + run: gitleaks detect --no-banner --redact --source . diff --git a/.gitleaks.toml b/.gitleaks.toml index a98099c..539fce4 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -14,6 +14,21 @@ useDefault = true [allowlist] description = "Unit-test dummy keys and documented placeholder keys (not real secrets)" +# Full-history scanning (new: the `gitleaks-history` job in +# security.yml, push-triggered only) sees commits the working-tree scan +# never could. It found exactly one hit across the whole history: a fixture +# in src/lib/client-factory.test.ts that once hardcoded a fake `tsp_u_` +# test value literally, before a later commit rewrote it to build the value +# at test-run time specifically so no token-shaped literal exists in source +# (see that file's own comment). Not a real credential — never minted by a +# real backend, never live. Allowlisted by COMMIT, not by value: writing the +# literal string itself into this file would trip +# scripts/make-public-snapshot.sh's own release-time leak grep, since this +# file ships to the public repo. Verified this is the only finding in the +# repo's history before adding the exemption. +commits = [ + "da04640835a2d9b02901c88581680ec2445c85b0", +] regexes = [ # Unit-test fixtures, e.g. sk-secret-12345 in *.test.ts '''sk-secret-[0-9]+''', diff --git a/CHANGELOG.md b/CHANGELOG.md index 33d23be..ad2d8da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +## [0.6.0] - 2026-08-12 + +### Added + +- **CI-native output now covers single-run `--wait` and batch rerun.** `--gh-output` / `--summary-file` (contributed in community PR #264 for `test run --all --wait`) now also work on `test run --wait` and `test rerun --all --wait`, so a CI job that runs one test or reruns a batch gets the same `::error::` annotations and machine summary as a fresh batch — written right after the JUnit report on the rerun path. The summary and annotations land even when the command exits non-zero, the emit is best-effort (a sink write failure such as EPIPE can never change the command's exit code), and non-dispatched work — rate-deferred, conflicted, or not-found tests — is folded into the summary as non-passed rows, so a partial batch can never read as "all passed". Workflow-command data and job-summary table cells are escaped: a multiline run error cannot inject `::commands` into the Actions stream or break the summary table. +- **`--target-url` now warns when it cannot apply (V3 path).** On a V3-routed account, `test run --target-url` is validated server-side and then discarded — V3 resolves the run's environment at execution time, so there is no per-run override. The CLI now makes a bounded, best-effort account check (only when the flag was supplied) and prints a stderr `[advisory]` that the flag will not take effect; `test create-batch --run` fires the same advisory once before the fan-out. It fires in every `--output` mode — JSON/CI is exactly the unattended case that needs it — and can never affect stdout or the exit code. + +### Fixed + +- **Create paths prefer the server's dashboard link and never print a dead one.** `test create` / `test create-batch` / `--plan-from` computed the Portal `dashboardUrl` entirely client-side in the legacy route shape, which 404s for a V3-native create. A server-provided `dashboardUrl` on the create response now wins, under the same three-state contract the run paths use: a string is used verbatim; `null` means the server looked and there is no correct link, so an `[advisory]` points at `test get ` instead of printing a URL that would 404; only an absent field (an older backend) falls back to the client computation. The decision is resolved once at create time and reused verbatim by the `--run` fan-out, and `test create --run` in text mode prints the server link again (its `Dashboard:` line had been dropped in the delegation to the run renderer). +- **Best-effort pre-flight lookups can no longer stall a real command behind a rate limit.** The duplicate-name advisory's 5-second deadline bounded the fetch but not the retry sleep, so a 429 carrying a long `Retry-After` could park an otherwise-healthy `test create` behind an advisory; the same applied to the `--target-url` account check. Both lookups now skip rate-limit retries entirely — a throttled advisory is dropped, never waited on. +- **Long `--wait` sessions no longer accumulate abort-signal churn.** Every poll iteration and every HTTP attempt composed a fresh abort signal against the process-lifetime shutdown signal, registering finalization-tracked dependents by the hundreds of thousands over a large batch — measured as multi-second stalls during engine cleanup passes. The poll loop now reuses one controller per session and the HTTP layer composes signals manually with listeners it removes synchronously; timeout, Ctrl-C, and retry behavior are unchanged. + ## [0.5.0] - 2026-08-05 ### Added diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 8fd3bbb..daca41e 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -505,11 +505,16 @@ testsprite test run --all --project proj_xxxxxxxx --wait \ # Optional custom suite name (default: testsprite:) testsprite test run --all --project proj_xxxxxxxx --wait \ --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json + +# GitHub-native CI output: ::error:: annotations + job-summary table + machine summary +testsprite test run test_xxxxxxxx --wait --summary-file ./summary.json --output json ``` Batch `--report` flags apply only to `test run --all --wait` (and batch `test rerun --wait`). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. Optional `--report-suite-name ` overrides the default `testsprite:` suite name. -`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. +**GitHub-native CI output** (contributed in [#264](https://github.com/TestSprite/testsprite-cli/pull/264)): when `GITHUB_ACTIONS=true`, any `test run --wait` (single test or `--all`) and any batch `test rerun --wait` additionally emit one `::error::` workflow-command line per non-passed run (annotating the PR checks tab) and append a Markdown results table to the job summary (`$GITHUB_STEP_SUMMARY`). Pass `--gh-output` to force the annotations outside Actions (previewable locally), and `--summary-file ` to also write the reduced machine summary JSON (`{total, passed, failed, timedOut, runs[]}`). Everything is written even when the command exits non-zero, and every write is best-effort — a failed write never changes the exit code. Tests that never dispatched (rate-deferred, conflicted, not found) appear as non-passed rows, so a partial batch cannot read as all-passed. Annotation and table content is escaped, so run-error text cannot inject workflow commands or break the table. + +`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. On a V3-routed account (`testsprite auth status` shows `routing: v3`), `--target-url` is currently **not applied** — V3 resolves the run's environment from the project configuration at execution time, so the run executes against the configured URL and the CLI prints an `[advisory]` on stderr saying so. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. #### `testsprite test rerun [test-id...]` @@ -551,6 +556,7 @@ Flags: - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. - `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). - `--report junit --report-file ` — with batch `--wait`, write a JUnit XML sidecar after polling (atomic write). Optional `--report-suite-name ` overrides the default `testsprite:` suite name. Requires `--wait`; not available on single-test reruns. +- `--gh-output` / `--summary-file ` — with batch `--wait`: GitHub-native CI output, same behavior as on `test run` (see above) — `::error::` annotations per non-passed run, a job-summary table under GitHub Actions, and the reduced machine summary JSON. Not available on single-test reruns. A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. diff --git a/README.md b/README.md index 23945b1..07e0559 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- - 🤖 **Agent-shaped output.** `test failure get` returns **one bundle** — the failing step, its neighbors, screenshots, DOM snapshots, the test source, a root-cause hypothesis, and a recommended fix target — all sharing a single `snapshotId`. The CLI _refuses_ to stitch data from two different runs, so an agent never reasons over a frankenstein context. - ♻️ **A loop, not a one-shot.** `create → run → failure get → fix → rerun` — every pass is banked, not thrown away. - 📐 **Scriptable & deterministic.** Stable `--output json` contract, predictable [exit codes](./DOCUMENTATION.md#exit-codes), and a `--dry-run` that exercises the full code path offline with canned data. +- 🚦 **CI-native.** On GitHub Actions, `--wait` runs annotate the PR checks tab with one `::error::` per failure and append a results table to the job summary — automatically. Add `--report junit` for a JUnit XML sidecar, `--summary-file` for a machine summary, or `--gh-output` to preview the annotations locally. [Details →](./DOCUMENTATION.md#run-commands) - 🔌 **One command to onboard your agent.** `testsprite agent install claude` drops a ready-made skill file into your repo so your coding agent knows how to drive the loop on its own. ## How it works diff --git a/eslint.security.config.mjs b/eslint.security.config.mjs new file mode 100644 index 0000000..facc3a0 --- /dev/null +++ b/eslint.security.config.mjs @@ -0,0 +1,50 @@ +/** + * Security-focused ESLint config used by the CI "Security" workflow only. + * Run: npx eslint src/ --config eslint.security.config.mjs + * + * Ported from community PR https://github.com/TestSprite/testsprite-cli/pull/220 + * (author @OkeyAmy) with the ignore list trimmed to this repo's actual + * conventions — see eslint.config.mjs for the shared ignore list. + */ +import security from 'eslint-plugin-security'; +import tseslint from 'typescript-eslint'; +import globals from 'globals'; + +export default tseslint.config( + { + ignores: ['dist/**', 'coverage/**', 'node_modules/**', 'perf/**', '.claude/worktrees/**'], + }, + { + files: ['**/*.ts', '**/*.mts', '**/*.cts'], + // `@typescript-eslint` is registered here (rule definitions only, none + // enabled) purely so this narrower config doesn't choke on the + // `// eslint-disable-next-line @typescript-eslint/...` directives that + // already exist in the tree for the MAIN lint pass — without this, + // ESLint reports "Definition for rule ... was not found" as an error on + // every such line under this config, which has nothing to do with + // security and would fail the job for unrelated reasons. + plugins: { ...security.configs.recommended.plugins, '@typescript-eslint': tseslint.plugin }, + languageOptions: { + parser: tseslint.parser, + ecmaVersion: 2022, + sourceType: 'module', + globals: { ...globals.node }, + }, + rules: { + ...security.configs.recommended.rules, + // Block non-literal paths in fs calls — catches CWE-22, CWE-73 + 'security/detect-non-literal-fs-filename': 'error', + // Warn on object injection via bracket notation with user input + 'security/detect-object-injection': 'warn', + // Warn on timing-unsafe comparisons (token equality checks) + 'security/detect-possible-timing-attacks': 'warn', + // Warn on non-literal RegExp (ReDoS) + 'security/detect-non-literal-regexp': 'warn', + // Error on child_process with non-literal args + 'security/detect-child-process': 'error', + // Disable/reduce noisy rules for a CLI codebase + 'security/detect-non-literal-require': 'off', + 'security/detect-unsafe-regex': 'warn', + }, + }, +); diff --git a/package-lock.json b/package-lock.json index 085287a..3b30a26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.5.0", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.5.0", + "version": "0.6.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", @@ -23,6 +23,7 @@ "ajv": "^8.20.0", "eslint": "^10.7.0", "eslint-config-prettier": "^9.1.0", + "eslint-plugin-security": "4.0.1", "globals": "^17.7.0", "msw": "^2.14.3", "prettier": "^3.3.3", @@ -2161,6 +2162,22 @@ "eslint": ">=7.0.0" } }, + "node_modules/eslint-plugin-security": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-security/-/eslint-plugin-security-4.0.1.tgz", + "integrity": "sha512-/lZCkOxPOWaf1jXAqgICrS8St3BMBccIPvhOSUYuV6VCr1o5nFVG998FnTLt6w2Nxb8Uo0nM8fzmnhp+GY/aEg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "safe-regex": "^2.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/eslint-scope": { "version": "9.1.2", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", @@ -3163,6 +3180,16 @@ "node": ">=6" } }, + "node_modules/regexp-tree": { + "version": "0.1.27", + "resolved": "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.27.tgz", + "integrity": "sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==", + "dev": true, + "license": "MIT", + "bin": { + "regexp-tree": "bin/regexp-tree" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -3235,6 +3262,16 @@ "fsevents": "~2.3.2" } }, + "node_modules/safe-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-2.1.1.tgz", + "integrity": "sha512-rx+x8AMzKb5Q5lQ95Zoi6ZbJqwCLkqi3XuJXp5P3rT8OEc6sZCJG5AE5dU3lsgRr/F4Bs31jSlVN+j5KrsGu9A==", + "dev": true, + "license": "MIT", + "dependencies": { + "regexp-tree": "~0.1.1" + } + }, "node_modules/semver": { "version": "7.7.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", @@ -3659,9 +3696,9 @@ } }, "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", + "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", "license": "MIT", "engines": { "node": ">=20.18.1" diff --git a/package.json b/package.json index e88b123..78009dd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.5.0", + "version": "0.6.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", @@ -70,6 +70,7 @@ "ajv": "^8.20.0", "eslint": "^10.7.0", "eslint-config-prettier": "^9.1.0", + "eslint-plugin-security": "4.0.1", "globals": "^17.7.0", "msw": "^2.14.3", "prettier": "^3.3.3", diff --git a/scripts/README.md b/scripts/README.md index 7e17420..92d9373 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -6,13 +6,14 @@ what (if anything) they're missing. Per DEV-356 ("Windows-proof the toolchain"): **the release/backport shell scripts are CI-only** — nothing in this directory requires a human to run bash or perl locally. -| File | Classification | Runs on | Notes | -| ------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `make-public-snapshot.sh` | **CI-ONLY** | the release pipeline's build job (`release-build.yml`) and the nightly divergence sentinel (dry-run mode) | Builds the scrubbed public snapshot. Never ships to the public repo (it's in its own DROP list). Not meant to be run by hand — see [`docs/internal/cli-oss/release-pipeline-ops.md`](../docs/internal/cli-oss/release-pipeline-ops.md) for the operator-facing flow. | -| `backport-public-pr.sh` | **CI-ONLY** (human-runnable for conflict recovery) | the auto-backport workflow, invoked per merged community PR | Also directly runnable by an operator resolving a cherry-pick conflict (`--record-only` after a manual fix) — that's an escape hatch, not the steady-state path. Requires `bash` + `gh` + `jq`; a Windows operator resolving a conflict does it via Git Bash/WSL, or asks another maintainer — this one script is the sole remaining bash dependency in the whole release flow. | -| `generate-version.mjs` | **HUMAN-RUN** (Node) | `npm run prebuild` / `npm run generate:version`, any OS | Pure Node — no shell-out, no bash/perl/BSD-vs-GNU assumptions. | -| `postbuild.mjs` | **HUMAN-RUN** (Node) | `npm run build`, any OS | Pure Node `fs` calls (sets the executable bit on `dist/index.js`; a no-op on Windows, which has no POSIX exec bit). | -| `p0-status-coverage.sql` | **DROPPED-INTERNAL** | ad hoc, by an operator, against Athena | Not part of any npm script or CI workflow. Never ships to the public repo (internal AWS resource references). | +| File | Classification | Runs on | Notes | +| ---------------------------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `make-public-snapshot.sh` | **CI-ONLY** | the release pipeline's build job (`release-build.yml`) and the nightly divergence sentinel (dry-run mode) | Builds the scrubbed public snapshot. Never ships to the public repo (it's in its own DROP list). Not meant to be run by hand — see [`docs/internal/cli-oss/release-pipeline-ops.md`](../docs/internal/cli-oss/release-pipeline-ops.md) for the operator-facing flow. | +| `backport-public-pr.sh` | **CI-ONLY** (human-runnable for conflict recovery) | the auto-backport workflow, invoked per merged community PR | Also directly runnable by an operator resolving a cherry-pick conflict (`--record-only` after a manual fix) — that's an escape hatch, not the steady-state path. Requires `bash` + `gh` + `jq`; a Windows operator resolving a conflict does it via Git Bash/WSL, or asks another maintainer — this one script is the sole remaining bash dependency in the whole release flow. | +| `generate-version.mjs` | **HUMAN-RUN** (Node) | `npm run prebuild` / `npm run generate:version`, any OS | Pure Node — no shell-out, no bash/perl/BSD-vs-GNU assumptions. | +| `postbuild.mjs` | **HUMAN-RUN** (Node) | `npm run build`, any OS | Pure Node `fs` calls (sets the executable bit on `dist/index.js`; a no-op on Windows, which has no POSIX exec bit). | +| `p0-status-coverage.sql` | **DROPPED-INTERNAL** | ad hoc, by an operator, against Athena | Not part of any npm script or CI workflow. Never ships to the public repo (internal AWS resource references). | +| `apply-security-settings.sh` | **DROPPED-INTERNAL** + **HUMAN-RUN, operator-only** (bash) | ad hoc, run by hand by a repo/org admin, with the shell's cwd inside the atlas checkout | Five GitHub/npm settings toggles: (1) public-repo secret scanning + push protection; (2a) atlas SHA-pinning enforcement ONLY — the selected-actions allowlist half is structurally impossible on a private repo (GitHub's `patterns_allowed` note: "only applies to public repositories") and is deliberately not attempted here; (2b) verifies the PUBLIC mirror's selected-actions allowlist covers the _public repo's own_ live workflows (fetched over the API, never from a local checkout — a local-checkout derivation would silently narrow the allowlist if run from a snapshot checkout missing atlas-only workflows); (4) CodeQL default setup on the public repo; (5) an atlas branch-ruleset force-push/deletion guard, with a full pre/post-PUT shape check and a timestamped local backup of the ruleset before every write. Plus (3) a read-only npm namespace-claim report. Refuses to run unless invoked with cwd inside a `testsprite-cli-atlas` checkout (checked via the git remote URL, not directory name) — it mutates atlas's own settings and has no meaning against any other checkout, including the public snapshot, which is why it's in `make-public-snapshot.sh`'s `DROP` list. Dry-run by default; `--apply` to mutate. Requires `bash` + `gh` (authenticated, admin rights) + `jq` + `npm` + `git`. Never runs `npm publish`. Not part of CI or any npm script — same family of out-of-band operator action as `docs/internal/cli-oss/supply-chain-hardening.md` §3. | ## The upshot for a Windows contributor / operator diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 82ca013..9b46a4f 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -867,6 +867,27 @@ describe('runWhoami', () => { expect(printed).toEqual(sampleMe); }); + // Confirms `auth whoami` / `auth status` never sends X-CLI-Command — it must + // stay a plain, untagged /me call (only `runInit`'s configure-validate step + // and `test run --target-url`'s v3Enabled probe tag this header). + it('sends no X-CLI-Command header', async () => { + writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const sentHeaders: Array | undefined> = []; + const capturingFetch = vi.fn( + async (_url: string, init: { headers?: Record }) => { + sentHeaders.push(init?.headers); + return meResponse(); + }, + ) as unknown as AuthDeps['fetchImpl']; + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: capturingFetch }, + ); + expect(sentHeaders).toHaveLength(1); + expect(sentHeaders[0]?.['x-cli-command']).toBeUndefined(); + }); + it('renders routing: v3 and the gap advisory when v3Enabled is true', async () => { writeProfile('default', { apiKey: 'sk-user-min' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index 80f8c73..a8d760b 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -87,6 +87,30 @@ describe('runDoctor — healthy environment', () => { expect(out).toContain('reached GET /me'); }); + // Confirms `doctor` never sends X-CLI-Command — it must stay a plain, + // untagged /me call (only `runInit`'s configure-validate step and + // `test run --target-url`'s v3Enabled probe tag this header). + it('sends no X-CLI-Command header on its GET /me check', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const sentHeaders: Array | undefined> = []; + const capturingFetch = vi.fn( + async (_url: string, init: { headers?: Record }) => { + sentHeaders.push(init?.headers); + return new Response(JSON.stringify(OK_ME), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }, + ) as unknown as DoctorDeps['fetchImpl']; + await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: capturingFetch }), ...deps }, + ); + expect(sentHeaders).toHaveLength(1); + expect(sentHeaders[0]?.['x-cli-command']).toBeUndefined(); + }); + it('adds a Routing check (v3) and the gap advisory when /me reports v3Enabled', async () => { writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 382c414..f200a8e 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -2,19 +2,66 @@ * Unit tests for `test rerun` — M3.4 piece-3. * * All HTTP is mocked via `makeFetch`. The polling loop's sleep injection is - * wired through `TestDeps.sleep` to avoid real delays. + * wired through `TestDeps.sleep` (`instantSleep` below) to avoid real delays + * — every test in this file already routes through it; there is no + * uninjected real sleep on the deferred-retry/polling paths. + * + * `testTimeout` is pinned below (rather than left at the vitest default) so + * a *future* regression that reintroduces a real, uninjected sleep on the + * rerun path fails within seconds instead of silently burning minutes. + * + * A splitting-by-seam mitigation was evaluated (batch/fan-out cases into + * their own file, matching `test.rerun.closure-fanout.spec.ts`'s existing + * precedent) and rejected: the slow CI runs for this file were not a real + * sleep or a vitest reporter-RPC timeout, but V8's `FinalizationRegistry`/ + * `WeakRef` cleanup (`JSFinalizationRegistry::Cleanup` → `KeepDuringJob` → + * `OrderedHashSet::Add`, confirmed via CPU stack sampling) processing a large + * backlog of kept-alive objects. Splitting made the stall *deterministic* + * (4/4 attempts) instead of occasional, because vitest's inter-file teardown + * forced the cleanup task to run right after the fan-out tests' churn. + * + * Two independent sources of that churn have since been found and fixed — + * do not re-attempt a file split as a mitigation for either without first + * checking whether a new one has appeared: + * + * 1. `poll.ts` minted a fresh `AbortController` + composed `AbortSignal` + * on every poll *iteration*, even though the abort target (an absolute + * deadline) never changes within a session. Fixed by hoisting the + * controller/signal to poll-session scope. + * 2. `http.ts::requestWithMeta` composed a fresh `AbortSignal.any([ + * timeoutSignal, options.signal?, shutdownSignal?])` on every HTTP + * *request* (per retry attempt). `shutdownSignal` defaults to the + * process-lifetime `globalShutdown.signal`, so this registered one more + * `FinalizationRegistry`-tracked dependent against that single + * long-lived signal per request — CI (Node 22) caught this directly: + * this file's `RATE_LIMITED in a closure member poll` test (every + * closure-member poll 429s, retried up to 3× by `http.ts`) hit this + * file's 10s `testTimeout` guard on CI while measuring 8ms locally, the + * same signature as the `poll.ts` stall. Fixed by `composeAbortSignals` + * (manual `AbortController` + explicit `addEventListener`/ + * `removeEventListener`, cleaned up synchronously per attempt) instead + * of the native `AbortSignal.any`. Measured on this file: `AbortSignal. + * any` calls against `globalShutdown.signal` dropped from ~330K to 94 + * (the remainder is `poll.ts`'s own per-session compositions). + * + * `testTimeout: 10_000` is what surfaced fix 2 above — it is doing its job. + * Kept at 10s rather than raised: a genuinely reintroduced real sleep (~10s + * per attempt) still blows the budget on its very first attempt. */ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import type { Command } from 'commander'; +import { describe, expect, it, vi } from 'vitest'; import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; import { ShutdownController } from '../lib/interrupt.js'; import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; import type { FetchImpl } from '../lib/http.js'; import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; +vi.setConfig({ testTimeout: 10_000 }); + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -5628,3 +5675,144 @@ describe('R-BAT: batch rerun --wait — InterruptError partial lists all dispatc expect(stderrBlock).toContain('run_b2'); }); }); + +// --------------------------------------------------------------------------- +// Gap B — CI-native output on batch rerun --wait (::error:: + summary file) +// --------------------------------------------------------------------------- + +describe('gh-output integration on batch rerun --wait (Gap B)', () => { + function mixedBatchHarness() { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const run1 = makeTerminalRun('run_b1', 'passed'); + run1.testId = 'test_1'; + const run2 = makeTerminalRun('run_b2', 'failed'); + run2.testId = 'test_2'; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_b1')) return { body: run1 }; + if (url.includes('/runs/run_b2')) return { body: run2 }; + return errorBody('NOT_FOUND'); + }); + return { creds, fetchImpl }; + } + + it('--gh-output --summary-file writes the reduced artifact + annotates the failed run (exit 1)', async () => { + const { creds, fetchImpl } = mixedBatchHarness(); + const dir = mkdtempSync(join(tmpdir(), 'cli-gh-output-rerun-')); + const summaryFile = join(dir, 'summary.json'); + const stdoutLines: string[] = []; + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'text', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + ghOutput: true, + summaryFile, + }, + { + ...creds, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + const artifact = JSON.parse(readFileSync(summaryFile, 'utf8')) as { + total: number; + passed: number; + failed: number; + runs: unknown[]; + }; + expect(artifact).toMatchObject({ total: 2, passed: 1, failed: 1 }); + // Forced annotations (off-Actions) land on the text stdout for the failed run only. + const annotations = stdoutLines.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(1); + expect(annotations[0]).toContain('test_2'); + }); + + it('under Actions with --output json: stdout stays parseable JSON, ::error:: goes to stderr', async () => { + const { creds, fetchImpl } = mixedBatchHarness(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + ghOutput: true, + }, + { + ...creds, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + env: { GITHUB_ACTIONS: 'true' } as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + // The batch envelope on stdout must remain parseable as-is. + const payload = JSON.parse(stdoutLines.join('\n')) as { accepted?: unknown[] }; + expect(Array.isArray(payload.accepted)).toBe(true); + expect(stdoutLines.some(line => line.startsWith('::error'))).toBe(false); + const annotations = stderrLines.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(1); + expect(annotations[0]).toContain('test_2'); + }); +}); + +describe('rerun --gh-output / --summary-file require a batch --wait (Gap B guard)', () => { + function disableExits(cmd: Command): void { + cmd.exitOverride(); + cmd.commands.forEach(disableExits); + } + + it('--gh-output on a single rerun (no --wait) → exit 5', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['rerun', 'test_1', '--gh-output'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('--gh-output on a batch without --wait → exit 5', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['rerun', 'test_1', 'test_2', '--gh-output'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index e0bcbe5..4a63239 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -4263,3 +4263,476 @@ describe('gh-output integration on run --all --wait (issue #99 reshape)', () => expect(stdoutLines.some(line => line.startsWith('::error'))).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// DEV-749 (client half) — --target-url advisory when the caller is V3-routed +// --------------------------------------------------------------------------- + +describe('runTestRun — DEV-749 --target-url V3 advisory', () => { + /** Routes GET /me to `me`, everything else to `rest`. */ + function fetchWithMe( + me: { v3Enabled?: boolean } | (() => never), + rest: (url: string) => { status?: number; body: unknown }, + ): typeof globalThis.fetch { + return makeFetch(url => { + if (url.endsWith('/me')) { + if (typeof me === 'function') me(); + return { body: me }; + } + return rest(url); + }); + } + + it('v3Enabled:true → prints the advisory on stderr, not stdout; exit code unaffected', async () => { + const { credentialsPath } = makeCreds(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const fetchImpl = fetchWithMe({ v3Enabled: true }, () => ({ body: TRIGGER_RESP })); + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(result).toMatchObject({ runId: 'run_abc' }); + expect(stderrLines.some(l => l.includes('--target-url') && l.includes('[advisory]'))).toBe( + true, + ); + // stdout must stay pure JSON — no advisory text, and it must still parse. + expect(() => JSON.parse(stdoutLines.join(''))).not.toThrow(); + expect(stdoutLines.join('')).not.toContain('[advisory]'); + }); + + it('tags the v3Enabled probe GET /me with X-CLI-Command: run-target-url-probe, and NOT the trigger POST', async () => { + const { credentialsPath } = makeCreds(); + const meHeaders: Array | undefined> = []; + const triggerHeaders: Array | undefined> = []; + const fetchImpl = makeFetch((url, init) => { + if (url.endsWith('/me')) { + meHeaders.push(init.headers as Record | undefined); + return { body: { v3Enabled: true } }; + } + triggerHeaders.push(init.headers as Record | undefined); + return { body: TRIGGER_RESP }; + }); + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + expect(meHeaders).toHaveLength(1); + expect(meHeaders[0]?.['x-cli-command']).toBe('run-target-url-probe'); + expect(triggerHeaders).toHaveLength(1); + expect(triggerHeaders[0]?.['x-cli-command']).toBeUndefined(); + }); + + it('v3Enabled:false → no advisory', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + const fetchImpl = fetchWithMe({ v3Enabled: false }, () => ({ body: TRIGGER_RESP })); + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.includes('--target-url'))).toBe(false); + }); + + it('v3Enabled absent (older backend) → no advisory', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + const fetchImpl = fetchWithMe({}, () => ({ body: TRIGGER_RESP })); + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.includes('--target-url'))).toBe(false); + }); + + it('no --target-url → GET /me is never called', async () => { + const { credentialsPath } = makeCreds(); + const meCalls: string[] = []; + const fetchImpl = fetchWithMe( + () => { + meCalls.push('called'); + throw new Error('unreachable'); + }, + () => ({ body: TRIGGER_RESP }), + ); + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + }, + ); + expect(meCalls).toHaveLength(0); + }); + + it('fires in --output text mode too (matches the C1 backend-test advisory: unconditional across modes)', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + const fetchImpl = fetchWithMe({ v3Enabled: true }, () => ({ body: TRIGGER_RESP })); + await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('--target-url'))).toBe( + true, + ); + }); + + it('/me lookup failure is swallowed — the run still succeeds and no advisory prints', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.endsWith('/me')) throw new Error('network blip'); + return { body: TRIGGER_RESP }; + }); + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(result).toMatchObject({ runId: 'run_abc' }); + expect(stderrLines.some(l => l.includes('--target-url'))).toBe(false); + }); + + it('--dry-run --target-url: the canned /me sample (v3Enabled:true) still demonstrates the advisory', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + const stdoutLines: string[] = []; + // No custom fetchImpl override: the dry-run client's own canned fetch + // answers GET /me with the real `me` sample (v3Enabled: true). + await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('--target-url'))).toBe( + true, + ); + expect(stdoutLines.join('')).not.toContain('[advisory]'); + }); + + // Finding 1 (dogfood 2026-08-09): `HttpClient.sleepBeforeRetry` (src/lib/http.ts) + // observes only the process-lifetime shutdown signal, never a per-request + // `AbortSignal` — so this probe's own 5s deadline could not interrupt a + // retry sleep. A retryable 429 carrying a real `Retry-After` (e.g. the + // production `inflight_cap` response) would previously stall the probe — + // and the run trigger behind it — for up to a minute. Fixed via + // `retryOnRateLimit: false` on the probe's `GET /me` call. + it('a 429 with a long Retry-After on the /me probe cannot delay the run trigger', async () => { + const { credentialsPath } = makeCreds(); + const stderrLines: string[] = []; + let meCallCount = 0; + const fetchImpl = (async (input: FetchInput) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.endsWith('/me')) { + meCallCount++; + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Too many requests', + nextAction: 'Wait Retry-After seconds and retry.', + requestId: 'req_probe', + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '60' } }, + ); + } + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; + + const startedAt = Date.now(); + const result = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: false, + timeoutSeconds: 60, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + const elapsedMs = Date.now() - startedAt; + + // The run still triggers promptly — the probe's failure must never + // block or delay it. + expect(result).toMatchObject({ runId: 'run_abc' }); + // Exactly one attempt: `retryOnRateLimit: false` makes the HTTP layer + // throw on the first 429 instead of sleeping out the (would-be) 60s + // Retry-After — proving the probe never entered a retry sleep at all. + // (Without the fix, HttpClient's real `setTimeout`-backed default sleep + // would block here for up to 60s per retry attempt.) + expect(meCallCount).toBe(1); + expect(elapsedMs).toBeLessThan(2_000); + // v3Enabled was never learned (the probe failed) — swallow-on-error + // still holds, no advisory printed either way. + expect(stderrLines.some(l => l.includes('--target-url'))).toBe(false); + }); +}); + +describe('gh-output integration on single-test run --wait (Gap A)', () => { + function singleRunHarness(run: RunResponse) { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + // Trigger POST: /tests/{id}/runs (not the /runs/{runId} poll). + if (url.includes('/tests/') && url.includes('/runs') && !url.includes('/runs/run_abc')) { + return { body: TRIGGER_RESP }; + } + return { body: run }; + }); + return { credentialsPath, fetchImpl }; + } + + it('--gh-output --summary-file writes the one-row artifact even though the run failed (exit 1)', async () => { + const { credentialsPath, fetchImpl } = singleRunHarness(makeFailedRun()); + const dir = mkdtempSync(join(tmpdir(), 'cli-gh-output-single-')); + const summaryFile = join(dir, 'summary.json'); + const stdoutLines: string[] = []; + const err = await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + ghOutput: true, + summaryFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + env: {} as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + const artifact = JSON.parse(readFileSync(summaryFile, 'utf8')) as { + total: number; + passed: number; + failed: number; + runs: unknown[]; + }; + expect(artifact).toMatchObject({ total: 1, passed: 0, failed: 1 }); + expect(artifact.runs).toHaveLength(1); + // Forced annotations (off-Actions) land on the text stdout for the failed run. + expect(stdoutLines.some(line => line.startsWith('::error') && line.includes('test_xyz'))).toBe( + true, + ); + }); + + it('under Actions with --output json: stdout stays parseable JSON, ::error:: goes to stderr', async () => { + const { credentialsPath, fetchImpl } = singleRunHarness(makeFailedRun()); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const err = await runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + ghOutput: true, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + env: { GITHUB_ACTIONS: 'true' } as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + // The run envelope on stdout must remain parseable as-is. + const printed = JSON.parse(stdoutLines.join('')) as Record; + expect(printed.status).toBe('failed'); + expect(stdoutLines.some(line => line.startsWith('::error'))).toBe(false); + const annotations = stderrLines.filter(line => line.startsWith('::error')); + expect(annotations).toHaveLength(1); + expect(annotations[0]).toContain('test_xyz'); + }); + + it('auto-enables under GITHUB_ACTIONS=true without --gh-output (no summary file needed)', async () => { + const { credentialsPath, fetchImpl } = singleRunHarness(makeFailedRun()); + const stdoutLines: string[] = []; + const err = await runTestRun( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + env: { GITHUB_ACTIONS: 'true' } as NodeJS.ProcessEnv, + sleep: instantSleep, + }, + ).catch(e => e); + expect(err).toMatchObject({ exitCode: 1 }); + expect(stdoutLines.some(line => line.startsWith('::error') && line.includes('test_xyz'))).toBe( + true, + ); + }); +}); + +describe('run --gh-output / --summary-file still require --wait (Gap A guard)', () => { + it('--gh-output without --wait → exit 5', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['run', 'test_xyz', '--gh-output'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('--summary-file without --wait → exit 5', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['run', 'test_xyz', '--summary-file', '/tmp/x.json'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index db4b169..d76ee5d 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -6276,6 +6276,69 @@ describe('runCreate', () => { expect(advisoryLine).toContain('test update'); }); + it('a 429 with a long Retry-After on the dup-name lookup cannot delay the create', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// test code'); + let listCallCount = 0; + let postCalled = false; + // Raw fetch impl rather than makeFetch, because this needs a real + // `Retry-After` response header. + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if ((init.method ?? 'GET') === 'GET' && url.includes('/tests')) { + listCallCount++; + return new Response( + JSON.stringify({ + error: { + code: 'RATE_LIMITED', + message: 'Too many requests', + nextAction: 'Wait Retry-After seconds and retry.', + requestId: 'req_dup', + }, + }), + { status: 429, headers: { 'content-type': 'application/json', 'retry-after': '60' } }, + ); + } + postCalled = true; + return new Response(JSON.stringify(SAMPLE_RESPONSE), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; + + const stderrLines: string[] = []; + const result = await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'frontend', + name: 'Sign-up happy', + codeFile, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + ); + + // The create still happens — a best-effort advisory must never gate it. + expect(result.testId).toBe('test_new'); + expect(postCalled).toBe(true); + // Exactly one attempt. The load-bearing assertion: `retryOnRateLimit: false` + // makes the 429 throw on the first response instead of entering a retry + // sleep. That sleep observes only the process shutdown signal, so the + // lookup's own 5s AbortController could not have interrupted it — without + // this, a 60s Retry-After honoured across 3 attempts would park the create + // behind the advisory for roughly two minutes. + expect(listCallCount).toBe(1); + // And nothing leaks to the user about it. + expect(stderrLines.filter(l => l.includes('[advisory]'))).toHaveLength(0); + }); + it('Fix 4 — swallows listing error and still proceeds with create', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('// test code'); @@ -9881,3 +9944,974 @@ describe('Fix 5 — dashboardUrl emission', () => { expect(out.join('').includes('dashboardUrl')).toBe(false); }); }); + +// --------------------------------------------------------------------------- +// DEV-737 — server-provided dashboardUrl precedence on create paths +// --------------------------------------------------------------------------- + +describe('DEV-737 — create paths prefer a server-provided dashboardUrl', () => { + function writeCodeFileDev737(contents: string): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dev737-')); + const path = join(dir, 'test.py'); + writeFileSync(path, contents, 'utf8'); + return path; + } + + it('runCreate JSON: a server-provided dashboardUrl wins over the client V2 guess', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileDev737('test("dash", async () => {});'); + const serverUrl = + 'https://www.testsprite.com/dashboard-v3/o/org_1/projects/proj_dash/test-cases/test_dash_01'; + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + testId: 'test_dash_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: serverUrl, + }, + }; + }); + const out: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => undefined }, + ); + const printed = JSON.parse(out.join('')) as { dashboardUrl?: string }; + // NOT the client-computed V2 shape (`/dashboard/tests/...`) — the server's + // V3 org-scoped link, verbatim. + expect(printed.dashboardUrl).toBe(serverUrl); + }); + + it('runCreate JSON: server dashboardUrl absent (backend predates this field) — falls back to the client-computed legacy link, no suppression advisory', async () => { + // Third state of the pinned three-state contract, alongside the + // present-string test above and the present-null test below: a wire + // response that OMITS the key entirely (not merely nullish) must be + // treated as "backend predates this field", never as suppression. + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileDev737('test("dash", async () => {});'); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + testId: 'test_dash_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + // No `dashboardUrl` key at all. + }, + }; + }); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + const printed = JSON.parse(out.join('')) as { dashboardUrl?: string }; + expect(printed.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/proj_dash/test/test_dash_01', + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('no dashboard link'))).toBe( + false, + ); + }); + + it('runCreate JSON: server dashboardUrl:null suppresses the link — no client guess, and an advisory fires', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileDev737('test("dash", async () => {});'); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + testId: 'test_dash_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: null, + }, + }; + }); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + const printed = JSON.parse(out.join('')) as Record; + // The field must be OMITTED, not serialized as a literal `null`. + expect('dashboardUrl' in printed).toBe(false); + expect( + stderrLines.some(l => l.includes('[advisory]') && l.includes('test get test_dash_01')), + ).toBe(true); + }); + + it('runCreate text mode: suppressed link — no dead Dashboard: line, advisory still fires on stderr', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileDev737('test("dash", async () => {});'); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + testId: 'test_dash_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: null, + }, + }; + }); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.startsWith('Dashboard:'))).toBe(false); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('no dashboard link'))).toBe( + true, + ); + }); + + it('runCreate --run chain: suppressed server link is not replaced by the client guess', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileDev737('test("dash", async () => {});'); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET' && url.includes('/tests?')) return { status: 200, body: { items: [] } }; + if (method === 'POST' && url.endsWith('/tests')) { + return { + status: 200, + body: { + testId: 'test_dash_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: null, + }, + }; + } + // POST /tests/{id}/runs — trigger + return { + status: 200, + body: { + runId: 'run_dash_01', + status: 'queued', + enqueuedAt: '2026-08-09T10:00:01.000Z', + codeVersion: 'v1', + targetUrl: '', + }, + }; + }); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_dash', + type: 'frontend', + name: 'dash test', + codeFile, + run: true, + wait: false, + timeout: 60, + timeoutIsDefault: true, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + const printed = JSON.parse(out.join('')) as Record; + expect('dashboardUrl' in printed).toBe(false); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('no dashboard link'))).toBe( + true, + ); + }); + + it('runCreateFromPlan JSON: a server-provided dashboardUrl wins over the plan-derived client guess', async () => { + function writePlanFileDev737(plan: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dev737-plan-')); + const path = join(dir, 'plan.json'); + writeFileSync(path, JSON.stringify(plan), 'utf8'); + return path; + } + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const planFile = writePlanFileDev737({ + projectId: 'proj_dash_plan', + type: 'frontend', + name: 'dash plan test', + planSteps: [{ type: 'action', description: 'navigate' }], + }); + const serverUrl = + 'https://www.testsprite.com/dashboard-v3/o/org_1/projects/proj_dash_plan/test-cases/test_dash_01'; + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + testId: 'test_dash_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: serverUrl, + }, + }; + }); + const out: string[] = []; + await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => undefined }, + ); + const printed = JSON.parse(out.join('')) as { dashboardUrl?: string }; + expect(printed.dashboardUrl).toBe(serverUrl); + }); + + it('runCreateFromPlan JSON: server dashboardUrl:null suppresses the link — no plan-derived client guess', async () => { + function writePlanFileDev737(plan: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dev737-plan-')); + const path = join(dir, 'plan.json'); + writeFileSync(path, JSON.stringify(plan), 'utf8'); + return path; + } + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const planFile = writePlanFileDev737({ + projectId: 'proj_dash_plan', + type: 'frontend', + name: 'dash plan test', + planSteps: [{ type: 'action', description: 'navigate' }], + }); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + testId: 'test_dash_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: null, + }, + }; + }); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + const printed = JSON.parse(out.join('')) as Record; + expect('dashboardUrl' in printed).toBe(false); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('no dashboard link'))).toBe( + true, + ); + }); + + it('runCreateFromPlan JSON: server dashboardUrl absent (backend predates this field) — falls back to the plan-derived client guess', async () => { + function writePlanFileDev737(plan: unknown): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dev737-plan-')); + const path = join(dir, 'plan.json'); + writeFileSync(path, JSON.stringify(plan), 'utf8'); + return path; + } + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const planFile = writePlanFileDev737({ + projectId: 'proj_dash_plan', + type: 'frontend', + name: 'dash plan test', + planSteps: [{ type: 'action', description: 'navigate' }], + }); + const fetchImpl = makeFetch((_url, init) => { + if ((init.method ?? 'GET') === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + testId: 'test_dash_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + // No `dashboardUrl` key at all. + }, + }; + }); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreateFromPlan( + { profile: 'default', output: 'json', debug: false, planFrom: planFile }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + const printed = JSON.parse(out.join('')) as { dashboardUrl?: string }; + expect(printed.dashboardUrl).toBe( + 'https://www.testsprite.com/dashboard/tests/proj_dash_plan/test/test_dash_01', + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('no dashboard link'))).toBe( + false, + ); + }); + + it('runCreateBatch JSON: per-item server dashboardUrl wins/suppresses independently; one aggregate advisory', async () => { + function writePlansJsonlDev737(plans: unknown[]): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dev737-batch-')); + const path = join(dir, 'plans.jsonl'); + writeFileSync(path, plans.map(p => JSON.stringify(p)).join('\n') + '\n', 'utf8'); + return path; + } + const specA = { + projectId: 'proj_batch_a', + type: 'frontend' as const, + name: 'batch spec a', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const specB = { + projectId: 'proj_batch_b', + type: 'frontend' as const, + name: 'batch spec b', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const plansFile = writePlansJsonlDev737([specA, specB]); + const serverUrlA = + 'https://www.testsprite.com/dashboard-v3/o/org_1/projects/proj_batch_a/test-cases/test_a'; + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const fetchImpl = makeFetch(() => ({ + status: 200, + body: { + results: [ + { specIndex: 0, status: 'created', testId: 'test_a', dashboardUrl: serverUrlA }, + { specIndex: 1, status: 'created', testId: 'test_b', dashboardUrl: null }, + ], + summary: { total: 2, created: 2, failed: 0 }, + }, + })); + const out: string[] = []; + const stderrLines: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + const printed = JSON.parse(out.join('')) as { + results: Array<{ testId: string; dashboardUrl?: string }>; + }; + const itemA = printed.results.find(r => r.testId === 'test_a')!; + const itemB = printed.results.find(r => r.testId === 'test_b')!; + expect(itemA.dashboardUrl).toBe(serverUrlA); + expect('dashboardUrl' in itemB).toBe(false); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('no dashboard link'))).toBe( + true, + ); + }); + + // R3b (finding 2) — the `--run` fan-out must reuse the SAME per-item + // dashboard decision the create phase already resolved, not recompute a + // client-side URL from testId→projectId. Covers both `--output json` + // (the field is directly assertable) and `--output text` (asserts no + // legacy URL leaks anywhere and the batch-run summary still renders). + function writeTwoSpecPlansDev737(): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-dev737-batch-run-')); + const path = join(dir, 'plans.jsonl'); + const specA = { + projectId: 'proj_run_a', + type: 'frontend' as const, + name: 'batch run spec a', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const specB = { + projectId: 'proj_run_b', + type: 'frontend' as const, + name: 'batch run spec b', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + writeFileSync(path, [specA, specB].map(p => JSON.stringify(p)).join('\n') + '\n', 'utf8'); + return path; + } + + const RUN_SERVER_URL_A = + 'https://www.testsprite.com/dashboard-v3/o/org_1/projects/proj_run_a/test-cases/test_run_a'; + const LEGACY_V2_URL_FRAGMENT = '/dashboard/tests/'; // the dead link this feature removes + + function makeBatchRunFanoutFetch(): typeof globalThis.fetch { + return makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + if (url.includes('/tests/batch')) { + return { + status: 200, + body: { + results: [ + { + specIndex: 0, + status: 'created', + testId: 'test_run_a', + dashboardUrl: RUN_SERVER_URL_A, + }, + { specIndex: 1, status: 'created', testId: 'test_run_b', dashboardUrl: null }, + ], + summary: { total: 2, created: 2, failed: 0 }, + }, + }; + } + // POST /tests/{id}/runs — trigger + return { + status: 200, + body: { + runId: 'run_fanout', + status: 'queued', + enqueuedAt: '2026-08-09T10:00:01.000Z', + codeVersion: 'v1', + targetUrl: '', + }, + }; + }); + } + + it('runCreateBatch --run --output json: per-item run results reuse the create-time server dashboardUrl/suppression — not a client-side recompute', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const plansFile = writeTwoSpecPlansDev737(); + const out: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + run: true, + wait: false, + dryRun: false, + }, + { + credentialsPath, + fetchImpl: makeBatchRunFanoutFetch(), + stdout: line => out.push(line), + stderr: () => undefined, + sleep: () => Promise.resolve(), + }, + ); + const printed = JSON.parse(out.join('')) as { + results: Array<{ testId: string; dashboardUrl?: string }>; + }; + const itemA = printed.results.find(r => r.testId === 'test_run_a')!; + const itemB = printed.results.find(r => r.testId === 'test_run_b')!; + // The server's V3-shaped link survives into the RUN result unchanged — + // a client-side recompute would have produced the legacy V2 shape instead. + expect(itemA.dashboardUrl).toBe(RUN_SERVER_URL_A); + expect(itemA.dashboardUrl).not.toContain(LEGACY_V2_URL_FRAGMENT); + // The suppressed item carries no link at all in its RUN result — a + // client-side recompute would have resurrected the dead legacy link here. + expect('dashboardUrl' in itemB).toBe(false); + expect(out.join('')).not.toContain(LEGACY_V2_URL_FRAGMENT); + }); + + it('runCreateBatch --run --output text: no legacy dashboard link leaks anywhere; batch-run summary still prints', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const plansFile = writeTwoSpecPlansDev737(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'text', + debug: false, + plans: plansFile, + run: true, + wait: false, + dryRun: false, + }, + { + credentialsPath, + fetchImpl: makeBatchRunFanoutFetch(), + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: () => Promise.resolve(), + }, + ); + // Text mode never prints per-item run dashboard links today, but the + // create-time suppression state must still be honored end to end: no + // dead legacy V2 link may appear anywhere in the output, and the + // create-phase aggregate advisory (now computed unconditionally, + // regardless of --output mode) must fire for the suppressed item. + expect(stdoutLines.join('\n')).not.toContain(LEGACY_V2_URL_FRAGMENT); + expect(stderrLines.join('\n')).not.toContain(LEGACY_V2_URL_FRAGMENT); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('no dashboard link'))).toBe( + true, + ); + expect(stderrLines.some(l => l.includes('batch-run summary:'))).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Finding 3 (dogfood 2026-08-09) — `test create --run` in text mode must still +// print the authoritative `Dashboard:` line. The create's own print is +// suppressed on the --run chain (delegated to `runTestRun` -> `printRunOrChain`, +// whose text-mode header renders via `renderCreateText`, which never prints +// `dashboardUrl`), so without an explicit emission the link silently +// disappears — worst on a no-wait V3 create, where the client cannot +// recompute it at all. +// --------------------------------------------------------------------------- + +describe('[finding-3] test create --run text mode prints the authoritative Dashboard: line', () => { + function writeCodeFileFinding3(contents: string): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-finding3-')); + const path = join(dir, 'test.py'); + writeFileSync(path, contents, 'utf8'); + return path; + } + + const TRIGGER_RESP_F3 = { + runId: 'run_f3_01', + status: 'queued' as const, + enqueuedAt: '2026-08-09T10:00:01.000Z', + codeVersion: 'v1', + targetUrl: '', + }; + + it('server dashboardUrl (string) → printed verbatim on stderr as Dashboard: ', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileFinding3('test("dash", async () => {});'); + const serverUrl = + 'https://www.testsprite.com/dashboard-v3/o/org_1/projects/proj_f3/test-cases/test_f3_01'; + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + if (method === 'POST' && url.endsWith('/tests')) { + return { + status: 200, + body: { + testId: 'test_f3_01', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: serverUrl, + }, + }; + } + // POST /tests/{id}/runs — trigger. + return { status: 200, body: TRIGGER_RESP_F3 }; + }); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_f3', + type: 'frontend', + name: 'f3 test', + codeFile, + run: true, + wait: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l === `Dashboard: ${serverUrl}`)).toBe(true); + }); + + it('server dashboardUrl:null → no Dashboard: line (no legacy guess); suppressed advisory fires instead', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileFinding3('test("dash", async () => {});'); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + if (method === 'POST' && url.endsWith('/tests')) { + return { + status: 200, + body: { + testId: 'test_f3_02', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: null, + }, + }; + } + return { status: 200, body: TRIGGER_RESP_F3 }; + }); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_f3', + type: 'frontend', + name: 'f3 test', + codeFile, + run: true, + wait: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.startsWith('Dashboard:'))).toBe(false); + expect( + stderrLines.some(l => l.includes('[advisory]') && l.includes('test get test_f3_02')), + ).toBe(true); + }); + + it('absent dashboardUrl key (backend that predates the field) → legacy client-computed link still prints', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileFinding3('test("dash", async () => {});'); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + if (method === 'POST' && url.endsWith('/tests')) { + return { + status: 200, + body: { + testId: 'test_f3_03', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + // No dashboardUrl key at all — simulates a backend that predates + // the field; the legacy client-side fallback is the correct + // behavior here (not suppression). + }, + }; + } + return { status: 200, body: TRIGGER_RESP_F3 }; + }); + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_f3', + type: 'frontend', + name: 'f3 test', + codeFile, + run: true, + wait: false, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: line => stderrLines.push(line), + }, + ); + expect( + stderrLines.some( + l => l === 'Dashboard: https://www.testsprite.com/dashboard/tests/proj_f3/test/test_f3_03', + ), + ).toBe(true); + }); + + it('--output json is unaffected (no duplicate Dashboard: line; field stays in the merged envelope)', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const codeFile = writeCodeFileFinding3('test("dash", async () => {});'); + const serverUrl = + 'https://www.testsprite.com/dashboard-v3/o/org_1/projects/proj_f3/test-cases/test_f3_04'; + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + if (method === 'POST' && url.endsWith('/tests')) { + return { + status: 200, + body: { + testId: 'test_f3_04', + type: 'frontend', + codeVersion: 'v1', + createdAt: '2026-08-09T10:00:00.000Z', + dashboardUrl: serverUrl, + }, + }; + } + return { status: 200, body: TRIGGER_RESP_F3 }; + }); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_f3', + type: 'frontend', + name: 'f3 test', + codeFile, + run: true, + wait: false, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.startsWith('Dashboard:'))).toBe(false); + const printed = JSON.parse(stdoutLines.join('')) as Record; + expect(printed['dashboardUrl']).toBe(serverUrl); + }); +}); + +// --------------------------------------------------------------------------- +// Finding 2 (dogfood 2026-08-09) — create-batch --run --target-url must still +// warn a V3-routed caller once, up front, even though this fan-out calls +// `triggerRunWithMeta` directly and bypasses `runTestRun` (where the +// `--target-url` V3 advisory normally lives). +// --------------------------------------------------------------------------- + +describe('[finding-2] create-batch --run --target-url V3 advisory fires once, not per item', () => { + function writeTwoSpecPlansFinding2(): string { + const dir = mkdtempSync(join(tmpdir(), 'cli-finding2-batch-run-')); + const path = join(dir, 'plans.jsonl'); + const specA = { + projectId: 'proj_f2_a', + type: 'frontend' as const, + name: 'finding2 spec a', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + const specB = { + projectId: 'proj_f2_b', + type: 'frontend' as const, + name: 'finding2 spec b', + planSteps: [{ type: 'action', description: 'navigate' }], + }; + writeFileSync(path, [specA, specB].map(p => JSON.stringify(p)).join('\n') + '\n', 'utf8'); + return path; + } + + /** Routes GET /me to `me`; POST /tests/batch to a 2-item created response; everything else to the run trigger. */ + function makeFinding2Fetch( + me: { v3Enabled?: boolean } | undefined, + meCalls: string[], + ): typeof globalThis.fetch { + return makeFetch(url => { + if (url.endsWith('/me')) { + meCalls.push('called'); + return { status: 200, body: me ?? {} }; + } + if (url.includes('/tests/batch')) { + return { + status: 200, + body: { + results: [ + { specIndex: 0, status: 'created', testId: 'test_f2_a' }, + { specIndex: 1, status: 'created', testId: 'test_f2_b' }, + ], + summary: { total: 2, created: 2, failed: 0 }, + }, + }; + } + // POST /tests/{id}/runs — trigger, once per created item. + return { + status: 200, + body: { + runId: `run_f2_${meCalls.length}`, + status: 'queued', + enqueuedAt: '2026-08-09T10:00:01.000Z', + codeVersion: 'v1', + targetUrl: '', + }, + }; + }); + } + + it('fires exactly once for the whole batch (not once per item) when V3-routed', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const plansFile = writeTwoSpecPlansFinding2(); + const meCalls: string[] = []; + const stderrLines: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'text', + debug: false, + plans: plansFile, + run: true, + wait: false, + dryRun: false, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl: makeFinding2Fetch({ v3Enabled: true }, meCalls), + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: () => Promise.resolve(), + }, + ); + // Exactly one /me probe for the whole batch, regardless of how many + // items are in it (two created + triggered here). + expect(meCalls).toHaveLength(1); + const advisoryLines = stderrLines.filter( + l => l.includes('[advisory]') && l.includes('--target-url'), + ); + expect(advisoryLines).toHaveLength(1); + }); + + it('does not fire when --target-url is absent, even for a V3-routed caller', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const plansFile = writeTwoSpecPlansFinding2(); + const meCalls: string[] = []; + const stderrLines: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'text', + debug: false, + plans: plansFile, + run: true, + wait: false, + dryRun: false, + }, + { + credentialsPath, + fetchImpl: makeFinding2Fetch({ v3Enabled: true }, meCalls), + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: () => Promise.resolve(), + }, + ); + // No --target-url supplied: the probe must not even fire (mirrors + // single `test run`'s "only pay the extra /me round trip when + // --target-url was actually supplied" gating). + expect(meCalls).toHaveLength(0); + expect(stderrLines.some(l => l.includes('--target-url'))).toBe(false); + }); + + it('v3Enabled:false → no advisory even with --target-url set', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const plansFile = writeTwoSpecPlansFinding2(); + const meCalls: string[] = []; + const stderrLines: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'text', + debug: false, + plans: plansFile, + run: true, + wait: false, + dryRun: false, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl: makeFinding2Fetch({ v3Enabled: false }, meCalls), + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: () => Promise.resolve(), + }, + ); + expect(meCalls).toHaveLength(1); + expect(stderrLines.some(l => l.includes('--target-url'))).toBe(false); + }); + + it('--output json: the advisory stays on stderr only — stdout parses clean with no [advisory] text', async () => { + const { credentialsPath } = makeCreds('sk-user-test', 'https://api.testsprite.com'); + const plansFile = writeTwoSpecPlansFinding2(); + const meCalls: string[] = []; + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + plans: plansFile, + run: true, + wait: false, + dryRun: false, + targetUrl: 'https://staging.example.com', + }, + { + credentialsPath, + fetchImpl: makeFinding2Fetch({ v3Enabled: true }, meCalls), + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: () => Promise.resolve(), + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('--target-url'))).toBe( + true, + ); + expect(() => JSON.parse(stdoutLines.join(''))).not.toThrow(); + expect(stdoutLines.join('')).not.toContain('[advisory]'); + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index b7fe1af..8898b07 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -97,7 +97,12 @@ import { import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; -import { emitGithubOutputs, summarizeAcceptedPayload } from '../lib/gh-output.js'; +import { emitTargetUrlV3Advisory } from '../lib/v3-advisory.js'; +import { + emitGithubOutputs, + summarizeAcceptedPayload, + summarizeSingleRun, +} from '../lib/gh-output.js'; import { loadConfig } from '../lib/config.js'; import { flakyExitCode, @@ -652,6 +657,18 @@ export interface CliCreateTestResponse { * still succeeded. */ warnings?: string[]; + /** + * Server-built Portal deep link for the created test (DEV-737). Same + * presence/absence contract as `RunResponse.dashboardUrl` + * (`withRunDashboardUrl` below): PRESENT (string or falsy) on a backend + * that resolved the question at all — a falsy value means "there is no + * correct link for this test" (e.g. a V3-native create with no DynamoDB + * mirror row for the client's V2-shaped guess to land on) and must never + * be replaced by the client fallback; ABSENT on an older backend that + * predates this field, which safely reopens the client fallback via + * `resolveDashboardUrl` below. See that function's doc for the reasoning. + */ + dashboardUrl?: string | null; } export const CLI_CREATE_PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; @@ -760,15 +777,26 @@ async function emitDupNameAdvisoryIfNeeded( // Use an AbortController with a 5 s deadline. When the timer fires it // calls ac.abort(), which causes client.get (via the `signal` option) to // throw an AbortError — caught below and swallowed. This ensures a stalled - // or retrying listing endpoint can't delay an otherwise-healthy create by - // the full request-timeout (120 s) or multiple transport retries. + // listing endpoint can't delay an otherwise-healthy create by the full + // request-timeout (120 s). // No secondary setTimeout is used to avoid leaking timers in tests. + // + // The deadline alone is NOT sufficient, because `signal` is composed into + // the fetch only — `sleepBeforeRetry` observes the process-lifetime + // shutdown signal and nothing else, so a retry sleep runs to completion + // no matter what this controller does. A 429 carrying `Retry-After` is + // honoured up to 60 s per attempt across 3 attempts, which would park the + // create behind a best-effort advisory for up to ~2 minutes. `429` is a + // live response for CLI callers (the in-flight run cap returns it), so + // this is reachable, not theoretical. `retryOnRateLimit: false` makes the + // 429 throw straight into the catch below, which is the correct outcome + // for a lookup whose entire purpose is to be skippable. const ac = new AbortController(); const timer = setTimeout(() => ac.abort(), DUP_NAME_ADVISORY_TIMEOUT_MS); try { const listing = await client.get<{ items: CliTest[] }>( `/tests?projectId=${encodeURIComponent(projectId)}&pageSize=100`, - { signal: ac.signal }, + { signal: ac.signal, retryOnRateLimit: false }, ); const nameLower = name.toLowerCase(); const match = listing.items?.find(t => t.name.toLowerCase() === nameLower); @@ -966,11 +994,36 @@ export async function runCreate( // the merged { ...createContext, run } envelope in JSON mode and // appears on the Dashboard: stderr line in text mode. // R1: suppress under --dry-run (fake canned test id). - const chainDashboardUrl = opts.dryRun - ? undefined - : resolvePortalUrl(resolveApiUrl(opts, deps), projectId, response.testId); - const createContextWithUrl = - chainDashboardUrl !== undefined ? { ...response, dashboardUrl: chainDashboardUrl } : response; + // DEV-737: prefer a server-provided dashboardUrl over the client guess + // (`withDashboardUrl`/`resolveDashboardUrl` above); when the server + // explicitly withheld one, never fall back to the client-computed + // V2-shaped link and tell the caller where to find the test instead. + const { entity: createContextWithUrl, suppressed: dashboardSuppressedOnRun } = withDashboardUrl( + response, + () => + opts.dryRun + ? undefined + : resolvePortalUrl(resolveApiUrl(opts, deps), projectId, response.testId), + ); + const runDashboardStderrFn = + deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + if (dashboardSuppressedOnRun) { + emitDashboardLinkSuppressedAdvisory(response.testId, runDashboardStderrFn); + } else if (opts.output !== 'json' && createContextWithUrl.dashboardUrl !== undefined) { + // Finding 3 (dogfood 2026-08-09): the merged create+run chain suppresses + // the create's own print (delegating to `runTestRun` -> `printRunOrChain`), + // whose text-mode header renders `createContext` via `renderCreateText` — + // which never prints `dashboardUrl` — and there is otherwise no stderr + // emission point for it on this chain (unlike the non-`--run` path below, + // which prints this same line explicitly). Without this, a text-mode + // `create --run` silently drops the authoritative link — worst on a + // no-wait V3 create, where the client cannot recompute it at all. Mirrors + // the non-run path's three-state handling exactly: a suppressed (`null`) + // server link takes the advisory branch above instead (no legacy guess); + // an absent key already resolved to the legacy client fallback (or + // `undefined` when unmapped) via `createContextWithUrl` above. + runDashboardStderrFn(`Dashboard: ${createContextWithUrl.dashboardUrl}`); + } await runTestRun( { ...opts, @@ -996,13 +1049,20 @@ export async function runCreate( // (no extra network call — both come from opts / response). // R1: suppress under --dry-run — the test id is a fake canned value // (e.g. "test_dryrun_create_2026") and a live-looking URL would mislead. - const dashboardUrl = opts.dryRun - ? undefined - : resolvePortalUrl(resolveApiUrl(opts, deps), projectId, response.testId); + // DEV-737: prefer a server-provided dashboardUrl over the client guess; + // never fall back when the server explicitly withheld one (see + // `withDashboardUrl`/`resolveDashboardUrl` above), and tell the caller + // where to find the test instead of printing a link that would 404. + const { entity: responseWithDashboardUrl, suppressed: dashboardSuppressed } = withDashboardUrl( + response, + () => + opts.dryRun + ? undefined + : resolvePortalUrl(resolveApiUrl(opts, deps), projectId, response.testId), + ); + const dashboardUrl = responseWithDashboardUrl.dashboardUrl ?? undefined; if (opts.output === 'json') { - out.print(dashboardUrl !== undefined ? { ...response, dashboardUrl } : response, data => - renderCreateText(data as CliCreateTestResponse), - ); + out.print(responseWithDashboardUrl, data => renderCreateText(data as CliCreateTestResponse)); } else { out.print(response, data => renderCreateText(data as CliCreateTestResponse)); if (dashboardUrl !== undefined) { @@ -1010,6 +1070,12 @@ export async function runCreate( stderrFn(`Dashboard: ${dashboardUrl}`); } } + if (dashboardSuppressed) { + emitDashboardLinkSuppressedAdvisory( + response.testId, + deps.stderr ?? (line => process.stderr.write(`${line}\n`)), + ); + } return response; } @@ -2012,6 +2078,12 @@ export interface CliBatchSpecResult { message: string; field?: string; }; + /** + * Server-built Portal deep link for this created item (DEV-737). Same + * presence/absence contract as `CliCreateTestResponse.dashboardUrl` — + * see that field's doc. + */ + dashboardUrl?: string | null; } export interface CliCreateBatchResponse { @@ -2043,6 +2115,13 @@ export interface CliBatchRunResult { failureKind?: string | null; /** Error envelope when the trigger itself failed (network/auth/validation). */ error?: { code: string; message: string; exitCode: number }; + /** + * Portal deep link (R3b), threaded through from the SAME per-item + * create-time decision `test create-batch`'s own output carries — never + * recomputed at run time. Absent when unresolvable OR explicitly + * suppressed by the server (see `resolveDashboardUrl`'s doc). + */ + dashboardUrl?: string; } /** Envelope emitted by `test create-batch --run` in JSON mode. */ @@ -2493,9 +2572,17 @@ export async function runCreateFromPlan( // Fix 5 (plan-from coverage): the projectId for the deep-link comes from // the validated PLAN body (not opts — `--plan-from` has no --project-id // flag). Same dry-run suppression as runCreate (fake canned test id). - const planDashboardUrl = opts.dryRun - ? undefined - : resolvePortalUrl(resolveApiUrl(opts, deps), plan.projectId, response.testId); + // DEV-737: prefer a server-provided dashboardUrl over the client guess + // (`withDashboardUrl`/`resolveDashboardUrl`, defined near + // `withRunDashboardUrl`); never fall back when the server explicitly + // withheld one, and tell the caller where to find the test instead. + const { entity: responseWithDashboardUrl, suppressed: planDashboardSuppressed } = + withDashboardUrl(response, () => + opts.dryRun + ? undefined + : resolvePortalUrl(resolveApiUrl(opts, deps), plan.projectId, response.testId), + ); + const planDashboardUrl = responseWithDashboardUrl.dashboardUrl ?? undefined; // --run chain (M3.3 piece-3): trigger + optionally wait. Per codex // round-1 P1: suppress the create's own print when chaining; @@ -2504,8 +2591,9 @@ export async function runCreateFromPlan( // Idempotency key for the run is the create key + ":run" suffix so a // retry of the whole chain gets the same runId. Per piece-3 spec. const runIdempotencyKey = `${idempotencyKey}:run`; - const createContextWithUrl = - planDashboardUrl !== undefined ? { ...response, dashboardUrl: planDashboardUrl } : response; + if (planDashboardSuppressed) { + emitDashboardLinkSuppressedAdvisory(response.testId, stderrFn); + } return runTestRun( { ...opts, @@ -2516,23 +2604,23 @@ export async function runCreateFromPlan( // first-run hint fires for `test create --plan-from --run --wait`. timeoutIsDefault: opts.timeoutIsDefault ?? false, wait: opts.wait === true, - createContext: createContextWithUrl, + createContext: responseWithDashboardUrl, }, deps, ).then(() => response); } if (opts.output === 'json') { - out.print( - planDashboardUrl !== undefined ? { ...response, dashboardUrl: planDashboardUrl } : response, - data => renderCreateText(data as CliCreateTestResponse), - ); + out.print(responseWithDashboardUrl, data => renderCreateText(data as CliCreateTestResponse)); } else { out.print(response, data => renderCreateText(data as CliCreateTestResponse)); if (planDashboardUrl !== undefined) { stderrFn(`Dashboard: ${planDashboardUrl}`); } } + if (planDashboardSuppressed) { + emitDashboardLinkSuppressedAdvisory(response.testId, stderrFn); + } return response; } @@ -3019,23 +3107,70 @@ export async function runCreateBatch( }); } - // Fix 5: enrich results with per-item dashboardUrl in JSON mode. + // Fix 5: enrich results with per-item dashboardUrl. // projectId comes from specs[specIndex].projectId; testId from the result row. // Only emitted where both are known client-side — no extra network calls. // R1: suppress under --dry-run — test ids are fake canned values and a // live-looking URL would mislead the caller. + // DEV-737: prefer a server-provided per-item dashboardUrl over the client + // guess (`withDashboardUrl`/`resolveDashboardUrl`, defined near + // `withRunDashboardUrl`); never fall back when the server explicitly + // withheld one for a given item — that would print exactly the dead + // V2-shaped link the server declined to emit. + // + // The per-item decision is captured into `testIdToDashboardState` — computed + // ONCE here, regardless of `--output` mode — so the `--run` fan-out below + // (`runBatchRun`) can reuse the SAME decision instead of recomputing a + // client-side URL from testId→projectId, which would silently replace a + // server-provided V3 link with the dead legacy V2 guess and lose + // suppression state entirely. Computing it unconditionally (not gated on + // `opts.output === 'json'`) also means the aggregate suppression advisory + // below now correctly fires in text mode too — it previously only ever + // fired in JSON mode because the per-item loop was skipped entirely in text + // mode, silently under-warning a text-mode caller. That matches the + // documented "advisory fires on stderr regardless of --output mode" + // convention this repo already follows for the single-`test create` advisory. const apiUrlForDashboard = resolveApiUrl(opts, deps); + let anyDashboardSuppressed = false; + const testIdToDashboardState = new Map< + string, + { dashboardUrl: string | undefined; suppressed: boolean } + >(); + if (!opts.dryRun) { + for (const r of response.results) { + if (r.status !== 'created' || r.testId === undefined) continue; + const testId = r.testId; + const spec = specs[r.specIndex]; + const projectId = spec?.projectId; + const { entity, suppressed } = withDashboardUrl(r, () => + projectId ? resolvePortalUrl(apiUrlForDashboard, projectId, testId) : undefined, + ); + if (suppressed) anyDashboardSuppressed = true; + testIdToDashboardState.set(testId, { + dashboardUrl: entity.dashboardUrl ?? undefined, + suppressed, + }); + } + } + if (anyDashboardSuppressed) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderrFn( + '[advisory] no dashboard link is available for one or more created tests right now; ' + + 'use `testsprite test get ` to look them up.', + ); + } + + // JSON-mode create output is enriched from the same map, so it stays + // byte-identical to before this refactor. const enrichedResponse: CliCreateBatchResponse = !opts.dryRun && opts.output === 'json' ? { ...response, results: response.results.map(r => { if (r.status !== 'created' || r.testId === undefined) return r; - const spec = specs[r.specIndex]; - const projectId = spec?.projectId; - if (!projectId) return r; - const dashboardUrl = resolvePortalUrl(apiUrlForDashboard, projectId, r.testId); - return dashboardUrl !== undefined ? { ...r, dashboardUrl } : r; + const state = testIdToDashboardState.get(r.testId); + if (!state) return r; + return { ...r, dashboardUrl: state.dashboardUrl }; }), } : response; @@ -3051,23 +3186,13 @@ export async function runCreateBatch( // --run: fan out a trigger for each created test, then emit results. if (opts.run === true) { - // R3b: build testId → projectId map from the create results + specs so - // runBatchRun can enrich per-item run JSON with dashboardUrl. - const runTestIdToProjectId = new Map(); - for (const r of response.results) { - if (r.status === 'created' && r.testId !== undefined) { - const projectId = specs[r.specIndex]?.projectId; - if (projectId) runTestIdToProjectId.set(r.testId, projectId); - } - } await runBatchRun( opts, response, client, out, deps, - opts.dryRun ? undefined : runTestIdToProjectId, - opts.dryRun ? undefined : apiUrlForDashboard, + opts.dryRun ? undefined : testIdToDashboardState, ); // runBatchRun handles its own exit-code logic via CLIError. // Return the create response to satisfy the return type; callers that @@ -3104,12 +3229,17 @@ async function runBatchRun( client: HttpClient, out: Output, deps: TestDeps, - /** R3b: testId → projectId mapping built from create results + specs, used to enrich - * run-path JSON items with dashboardUrl. Populated by the caller; absent (undefined) - * means no enrichment (e.g. dry-run or caller didn't supply it). */ - testIdToProjectId?: Map, - /** R3b: resolved API URL for portal link resolution. */ - apiUrlForDashboard?: string, + /** + * R3b: per-testId dashboard-link decision, ALREADY resolved at create time + * via `withDashboardUrl`/`resolveDashboardUrl` (the shared three-state + * precedence helper — see its doc). Populated by the caller; absent + * (undefined) means no enrichment (e.g. dry-run). Reusing this decision + * — rather than recomputing a client-side URL from testId→projectId here + * — is the fix: a fresh `resolvePortalUrl` call at this point would + * silently replace a server-provided V3 link with the dead legacy V2 + * guess, and would have no way to know a link was explicitly suppressed. + */ + testIdToDashboardState?: Map, ): Promise { const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const timeoutSeconds = opts.timeoutSeconds ?? DEFAULT_RUN_TIMEOUT_SECONDS; @@ -3125,6 +3255,21 @@ async function runBatchRun( return; } + // Finding 2 (dogfood 2026-08-09): this fan-out calls `triggerRunWithMeta` + // directly (below), bypassing `runTestRun` entirely — so `runTestRun`'s own + // `--target-url` V3 advisory (`emitTargetUrlV3AdvisoryIfNeeded`) never fired + // here, even though every triggered run in the batch ignores `--target-url` + // exactly like a single V3-routed `test run` does. Reuses the same helper + // and advisory text (no duplicated probe logic or copy) and fires exactly + // ONCE for the whole batch, before the fan-out — not per item. Mirrors + // `runTestRun`'s own gating (only pay the extra `/me` round trip when + // `--target-url` was actually supplied) and its "fires under --dry-run too" + // behavior (the dry-run client's canned `/me` sample demonstrates the same + // advisory at zero real-network cost). + if (opts.targetUrl !== undefined) { + await emitTargetUrlV3AdvisoryIfNeeded(client, stderrFn); + } + // Dry-run: print a descriptor envelope and return without real triggers. if (opts.dryRun) { const dryRunResults: CliBatchRunResult[] = testIds.map(testId => ({ @@ -3583,16 +3728,19 @@ async function runBatchRun( // Emit output. if (opts.output === 'json') { - // R3b: enrich per-item run results with dashboardUrl when both testId and - // projectId are known (from the testIdToProjectId map built by the caller). - // Additive-optional: items where projectId is unknown are left unchanged. + // R3b: enrich per-item run results with the dashboard state ALREADY + // resolved at create time (`testIdToDashboardState`, built by the + // caller). Never recompute a client-side URL here — that would silently + // replace a server-provided V3 link with the dead legacy V2 guess, and + // would have no way to represent "the server explicitly suppressed + // this". A suppressed item (dashboardUrl undefined, suppressed: true) + // is left with no key at all, same as the create-time convention. const enrichedResults = - !opts.dryRun && testIdToProjectId !== undefined && apiUrlForDashboard !== undefined + !opts.dryRun && testIdToDashboardState !== undefined ? batchRunResults.map(r => { - const projectId = testIdToProjectId.get(r.testId); - if (!projectId || !r.testId) return r; - const dashboardUrl = resolvePortalUrl(apiUrlForDashboard, projectId, r.testId); - return dashboardUrl !== undefined ? { ...r, dashboardUrl } : r; + const state = testIdToDashboardState.get(r.testId); + if (!state || state.dashboardUrl === undefined) return r; + return { ...r, dashboardUrl: state.dashboardUrl }; }) : batchRunResults; out.print({ results: enrichedResults }); @@ -5545,6 +5693,14 @@ interface RunTestRunOptions extends CommonOptions { */ timeoutIsDefault?: boolean; idempotencyKey?: string; + /** + * --gh-output: force the GitHub-native output layer (::error:: annotations + + * job-summary table) even off-Actions. On the single-test `--wait` path the + * result is reduced to a one-row CI summary (`summarizeSingleRun`). + */ + ghOutput?: boolean; + /** --summary-file: also write the reduced machine summary JSON to this path. */ + summaryFile?: string; /** * Per codex round-1 P1: when chained from `test create --run`, the caller * passes the create response here so `runTestRun` can emit a single merged @@ -5632,6 +5788,13 @@ interface RunTestRerunOptions extends CommonOptions { reportFile?: string; /** --report-suite-name: optional override for the JUnit . */ reportSuiteName?: string; + /** + * --gh-output: force the GitHub-native output layer (::error:: annotations + + * job-summary table) on the batch-rerun `--wait` result, even off-Actions. + */ + ghOutput?: boolean; + /** --summary-file: also write the reduced machine summary JSON to this path. */ + summaryFile?: string; } /** @@ -5833,6 +5996,86 @@ function printRunOrChain( }); } +/** + * Server-first precedence for a `dashboardUrl` field on any wire response + * that can carry one (`RunResponse`, `CliCreateTestResponse`, + * `CliBatchSpecResult`). Single source of truth for the rule so the create + * paths (DEV-737) and the run-completion path (`withRunDashboardUrl` below) + * can't drift apart. + * + * This is a PINNED three-state wire contract — a prior revision let a + * present-but-falsy value and a truly-absent key both mean "no server + * opinion," which was ambiguous with the real backend behavior of omitting + * the key on a deliberate "no correct link" response, printing the dead + * legacy link this feature exists to remove. The backend now always + * includes the key (typed `string | null`, never omitted when it has an + * opinion), so the three states below are mutually exclusive on the wire: + * + * - **Present + truthy** → the server built a correct link (it alone knows + * which store answered and this environment's portal origin); use it + * verbatim. + * - **Present + falsy (`null`/`''`)** → the server explicitly has no + * correct link — e.g. a V3-native entity with no DynamoDB mirror row for + * the client's V2-shaped `/dashboard/tests/…` guess to land on. NEVER + * fall back here: a client-side guess would be exactly the dead link the + * server declined to emit. `suppressed: true` tells the caller a link + * was actively withheld (vs. simply never computed) so it can point the + * user elsewhere instead of printing nothing unexplained. + * - **Absent** → an older backend that predates this field entirely. Such a + * backend cannot have produced a V3-native/unmirrored entity either — + * that capability and this field ship together — so the client fallback + * is safe and reproduces exactly today's behavior. + */ +function resolveDashboardUrl( + wire: { dashboardUrl?: string | null }, + computeFallback: () => string | undefined, +): { dashboardUrl: string | undefined; suppressed: boolean } { + if ('dashboardUrl' in wire) { + return wire.dashboardUrl + ? { dashboardUrl: wire.dashboardUrl, suppressed: false } + : { dashboardUrl: undefined, suppressed: true }; + } + return { dashboardUrl: computeFallback(), suppressed: false }; +} + +/** + * Applies {@link resolveDashboardUrl} to a whole entity, returning a + * NORMALIZED copy: a server-suppressed (`null`/`''`) value is rewritten to + * `undefined` so `JSON.stringify` omits the key entirely instead of + * serializing a literal `"dashboardUrl": null` — the same normalization + * `withRunDashboardUrl` already did inline for `RunResponse`, generalized + * here so the create paths (DEV-737) can share it byte-for-byte. + */ +function withDashboardUrl( + wire: T, + computeFallback: () => string | undefined, +): { entity: T; suppressed: boolean } { + const { dashboardUrl, suppressed } = resolveDashboardUrl(wire, computeFallback); + if ('dashboardUrl' in wire) return { entity: { ...wire, dashboardUrl }, suppressed }; + return { entity: dashboardUrl !== undefined ? { ...wire, dashboardUrl } : wire, suppressed }; +} + +/** + * DEV-737: advisory printed when the server explicitly withheld a dashboard + * link (`resolveDashboardUrl`'s `suppressed: true`) rather than silently + * omitting the field with no explanation. Fires on stderr regardless of + * `--output` mode — the field's absence from a JSON envelope carries no + * reason on its own, and `--output json` is routinely unattended, so the + * hint is worth the line there too (same reasoning as the `project create + * --type backend` no-URL advisory in `project.ts`). Printing nothing beats + * printing a dead link, but telling the caller where to look instead beats + * printing nothing unexplained. + */ +function emitDashboardLinkSuppressedAdvisory( + testId: string, + stderrFn: (line: string) => void, +): void { + stderrFn( + `[advisory] no dashboard link is available for this test right now; use ` + + `\`testsprite test get ${testId}\` to look it up.`, + ); +} + /** * Attach the Portal deep link to a terminal RunResponse. * @@ -5854,17 +6097,10 @@ function printRunOrChain( * backend chose not to emit. */ function withRunDashboardUrl(run: RunResponse, apiUrl: string): RunResponse { - // A server value of `null` is meaningful, not missing: it says "there is no - // correct link for this run" (e.g. a workspace-scoped page the environment's - // portal build does not serve yet). Falling back to our own guess there would - // reintroduce exactly the wrong link the server declined to send — so only an - // ABSENT field (older backend) reopens the client path. - if ('dashboardUrl' in run) { - return run.dashboardUrl ? run : { ...run, dashboardUrl: undefined }; - } - if (!run.projectId || !run.testId) return run; - const dashboardUrl = resolvePortalUrl(apiUrl, run.projectId, run.testId); - return dashboardUrl !== undefined ? { ...run, dashboardUrl } : run; + return withDashboardUrl(run, () => { + if (!run.projectId || !run.testId) return undefined; + return resolvePortalUrl(apiUrl, run.projectId, run.testId); + }).entity; } /** @@ -5984,6 +6220,82 @@ function parseTimeoutFlag(raw: string | undefined, flagName: string): number { return n; } +/** + * Short deadline for the best-effort `v3Enabled` lookup behind the + * `--target-url` advisory (DEV-749). Mirrors `DUP_NAME_ADVISORY_TIMEOUT_MS` + * — a stalled `/me` must never meaningfully delay a run trigger. + */ +const TARGET_URL_ADVISORY_TIMEOUT_MS = 5_000; + +/** + * DEV-749 (client half): `--target-url` is silently dropped on the V3 + * execution path — the backend validates it (SSRF guard, defence in depth) + * and then discards it, because V3 resolves the run's environment from + * `test_environment` at execution time; there is no injection point for a + * per-run override. This is a client-side advisory only — the real fix + * (an actual per-run override on V3) needs a product decision and is out + * of scope here. + * + * Gated on `v3Enabled`, the same authoritative per-user routing bit + * `auth status`/`doctor` already render (`GET /me`, `src/lib/v3-advisory.ts`). + * Best-effort: a single bounded lookup (mirrors `emitDupNameAdvisoryIfNeeded`'s + * `AbortController` + 5 s deadline), swallows every error, and must never + * block or fail the actual run trigger — a broken/slow `/me` degrades to + * "no advisory printed", not a failed `test run`. + */ +/** + * `X-CLI-Command` tag for this probe's `GET /me` — same advisory-header + * mechanism `runInit` uses (`toAuthDeps`'s `commandTag: 'init'` in + * `commands/init.ts`, sent via `AuthDeps.commandTag` in `auth.ts`), reused + * directly here rather than invented fresh. The backend's CLI-audit + * allowlist (`KNOWN_CLI_COMMANDS` + `resolveCliEvent`'s `/me` branch) maps this + * exact string to `null` — no PostHog event at all, not `cli.session_started` + * and not `cli.initialized` (init's tag would be a MORE specific, and equally + * wrong, misattribution of a probe that is neither a session start nor an + * onboarding run). + * + * Forward-compatible by construction: an unlisted `X-CLI-Command` value is + * dropped by the backend before event selection (see + * `capturePostHogEvent`/`KNOWN_CLI_COMMANDS` in the interceptor above), so + * against a backend that predates this allowlist entry the header is a no-op + * and behavior is exactly what it was before this fix — the CLI can ship + * this independently of backend deploy order in either direction. + */ +const TARGET_URL_ADVISORY_CLI_COMMAND = 'run-target-url-probe'; + +async function emitTargetUrlV3AdvisoryIfNeeded( + client: HttpClient, + stderrFn: (line: string) => void, +): Promise { + const ac = new AbortController(); + const timer = setTimeout(() => ac.abort(), TARGET_URL_ADVISORY_TIMEOUT_MS); + try { + const me = await client.get<{ v3Enabled?: boolean }>('/me', { + signal: ac.signal, + headers: { 'x-cli-command': TARGET_URL_ADVISORY_CLI_COMMAND }, + // Finding 1 (dogfood 2026-08-09): `HttpClient.sleepBeforeRetry` (src/lib/http.ts) + // only observes the process-lifetime shutdown signal, never a per-request + // `options.signal` — the abort controller above bounds the FETCH itself but + // not a post-response retry sleep. A retryable 429 carrying a real + // `Retry-After` (e.g. 60s, a live `inflight_cap` response) would put this + // best-effort probe into a retry sleep this 5s deadline cannot interrupt, + // stalling the real run trigger behind it. `retryOnRateLimit: false` makes + // the HTTP layer throw on the first 429 instead of sleeping — the existing, + // already-used-elsewhere knob (see the batch-run trigger site above) rather + // than changing `sleepBeforeRetry` itself, which every other caller (e.g. + // the polling path) relies on to keep sleeping through a caller signal. + retryOnRateLimit: false, + }); + if (me.v3Enabled === true) { + emitTargetUrlV3Advisory(stderrFn); + } + } catch { + // Swallow — this is best-effort; must not block or fail the run trigger. + } finally { + clearTimeout(timer); + } +} + /** * `test run ` — M3.3 piece-3. * @@ -6001,6 +6313,16 @@ export async function runTestRun( assertNotLocal(opts.targetUrl); } + // DEV-749: only spend the extra `/me` round trip when --target-url was + // actually supplied — the common case (no override) pays nothing. Fires + // in both the dry-run and real paths below (the dry-run `/me` sample sets + // `v3Enabled: true`, so `--dry-run --target-url` demonstrates the same + // advisory a real V3-routed caller would see, at zero real-network cost). + if (opts.targetUrl !== undefined) { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + await emitTargetUrlV3AdvisoryIfNeeded(makeClient(opts, deps), stderrFn); + } + if (opts.dryRun) { const client = makeClient(opts, deps); const out = makeOutput(opts.output, deps); @@ -6365,11 +6687,9 @@ export async function runTestRun( beFallbackUsed || opts.type === 'backend', ); - printRunOrChain( - out, - withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), - opts.createContext, - data => renderRunResponseText(data as RunResponse, { isBackend }), + const finalRunWithUrl = withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)); + printRunOrChain(out, finalRunWithUrl, opts.createContext, data => + renderRunResponseText(data as RunResponse, { isBackend }), ); // Surface the trigger requestId under --verbose/--debug or JSON mode so @@ -6388,6 +6708,45 @@ export async function runTestRun( ); } + // CI-native output layer (issue #99): single-test parity with the --all batch. + // Emitted before the exit-code gate throws below so the summary file and + // annotations land even when the run exits non-zero (mirrors the batch path). + // The summary file is a machine artifact written regardless of --output mode; + // under --output json the run envelope above owns stdout, so ::error:: + // workflow commands are routed to stderr (the Actions runner parses both). + // Best-effort: a sink write throwing (e.g. EPIPE) must never skip the + // exit-code gate below or change the command's exit status. + try { + const env = deps.env ?? process.env; + const ghEnabled = opts.ghOutput === true || env.GITHUB_ACTIONS === 'true'; + if (ghEnabled || opts.summaryFile !== undefined) { + const ciSummary = summarizeSingleRun(finalRunWithUrl); + if (opts.summaryFile !== undefined) { + try { + writeFileSync(opts.summaryFile, `${JSON.stringify(ciSummary, null, 2)}\n`, 'utf8'); + } catch { + stderrFn(`[run] could not write --summary-file ${opts.summaryFile}; continuing`); + } + } + if (ghEnabled) { + const stdoutFn = deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)); + emitGithubOutputs( + ciSummary, + env, + { + stdout: stdoutFn, + stderr: stderrFn, + appendFile: (path: string, content: string) => appendFileSync(path, content, 'utf8'), + annotations: opts.output === 'json' ? stderrFn : stdoutFn, + }, + { force: opts.ghOutput === true }, + ); + } + } + } catch (ciErr) { + stderrFn(`[run] CI output emission failed; continuing: ${(ciErr as Error).message}`); + } + const exitCode = exitCodeForRunStatus(finalRun.status); if (exitCode !== 0) { // Throw a CLIError so index.ts exits with the right code without @@ -9180,6 +9539,39 @@ export async function runTestRerun( }; await writeBatchJUnitReportIfRequested(opts, rerunResults); out.print(jsonPayload); + // CI-native output layer (issue #99): batch-rerun parity with `run --all`. + // Emitted before the exit-code gates below so the summary file / annotations + // land even when the batch exits non-zero. Summary-file is a machine artifact + // written regardless of --output mode; under --output json the envelope above + // owns stdout, so ::error:: workflow commands go to stderr instead. + { + const env = deps.env ?? process.env; + const ghEnabled = opts.ghOutput === true || env.GITHUB_ACTIONS === 'true'; + if (ghEnabled || opts.summaryFile !== undefined) { + const ciSummary = summarizeAcceptedPayload(JSON.stringify(jsonPayload)); + if (opts.summaryFile !== undefined) { + try { + writeFileSync(opts.summaryFile, `${JSON.stringify(ciSummary, null, 2)}\n`, 'utf8'); + } catch { + stderrFn(`[rerun] could not write --summary-file ${opts.summaryFile}; continuing`); + } + } + if (ghEnabled) { + const stdoutFn = deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)); + emitGithubOutputs( + ciSummary, + env, + { + stdout: stdoutFn, + stderr: stderrFn, + appendFile: (path: string, content: string) => appendFileSync(path, content, 'utf8'), + annotations: opts.output === 'json' ? stderrFn : stdoutFn, + }, + { force: opts.ghOutput === true }, + ); + } + } + } // Determine exit code: timeout (deferred or any timeout) → 7; any fail → 1; all pass → 0 if (deferred.length > 0 || timedOut > 0) { @@ -10076,11 +10468,11 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--gh-output', - 'with --all --wait: emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true', + 'with --wait (single test or --all): emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true', ) .option( '--summary-file ', - 'with --all --wait: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file', + 'with --wait (single test or --all): also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file', ) .addHelpText( 'after', @@ -10130,20 +10522,21 @@ export function createTestCommand(deps: TestDeps = {}): Command { wait: cmdOpts.wait === true, batchPath: isAll, }); - // --gh-output / --summary-file reduce the terminal batch envelope, which - // only exists on the --all --wait path (without --wait the command returns - // after enqueueing). Anywhere else they would silently no-op — reject - // loudly (same rule as --filter and the JUnit report flags). - if (cmdOpts.ghOutput === true && (!isAll || cmdOpts.wait !== true)) { + // --gh-output / --summary-file reduce a --wait run's terminal result into + // the CI summary. They require --wait (without it the command returns after + // enqueueing, before any terminal result exists) but apply to BOTH a single + // run and the --all batch. Without --wait they would silently + // no-op — reject loudly (same rule as --filter and the JUnit report flags). + if (cmdOpts.ghOutput === true && cmdOpts.wait !== true) { throw localValidationError( 'gh-output', - '--gh-output only applies with --all --wait (it reduces the terminal batch envelope). Remove --gh-output, or add --all --wait.', + '--gh-output requires --wait (it reduces the terminal run result). Add --wait.', ); } - if (cmdOpts.summaryFile !== undefined && (!isAll || cmdOpts.wait !== true)) { + if (cmdOpts.summaryFile !== undefined && cmdOpts.wait !== true) { throw localValidationError( 'summary-file', - '--summary-file only applies with --all --wait (it reduces the terminal batch envelope). Remove --summary-file, or add --all --wait.', + '--summary-file requires --wait (it reduces the terminal run result). Add --wait.', ); } @@ -10199,6 +10592,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { // B2(c): tell runTestRun whether --timeout was explicitly provided. timeoutIsDefault: cmdOpts.timeout === undefined, idempotencyKey: cmdOpts.idempotencyKey, + ghOutput: cmdOpts.ghOutput === true, + summaryFile: cmdOpts.summaryFile, }, deps, ); @@ -10336,6 +10731,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--report-suite-name ', 'optional JUnit override (default: testsprite:)', ) + .option( + '--gh-output', + 'with batch --wait: emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true', + ) + .option( + '--summary-file ', + 'with batch --wait: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file', + ) .addHelpText( 'after', '\nNotes:\n' + @@ -10373,6 +10776,21 @@ export function createTestCommand(deps: TestDeps = {}): Command { wait: cmdOpts.wait === true, batchPath: isBatch, }); + // --gh-output / --summary-file reduce the batch-rerun --wait envelope, which + // only exists on the batch (--all or 2+ ids) --wait path. Anywhere else they + // would silently no-op — reject loudly (same rule as the JUnit report flags). + if (cmdOpts.ghOutput === true && (!isBatch || cmdOpts.wait !== true)) { + throw localValidationError( + 'gh-output', + '--gh-output requires a batch rerun with --wait (--all or 2+ test ids). Remove --gh-output, or add --all --wait.', + ); + } + if (cmdOpts.summaryFile !== undefined && (!isBatch || cmdOpts.wait !== true)) { + throw localValidationError( + 'summary-file', + '--summary-file requires a batch rerun with --wait (--all or 2+ test ids). Remove --summary-file, or add --all --wait.', + ); + } await runTestRerun( { ...resolveCommonOptions(command), @@ -10394,6 +10812,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { report, reportFile: cmdOpts.reportFile, reportSuiteName: cmdOpts.reportSuiteName, + ghOutput: cmdOpts.ghOutput === true, + summaryFile: cmdOpts.summaryFile, }, deps, ); @@ -10724,6 +11144,8 @@ interface RerunFlagOpts { report?: string; reportFile?: string; reportSuiteName?: string; + ghOutput?: boolean; + summaryFile?: string; } interface UpdateFlagOpts { diff --git a/src/commands/usage.test.ts b/src/commands/usage.test.ts index 71fd4b1..9784df4 100644 --- a/src/commands/usage.test.ts +++ b/src/commands/usage.test.ts @@ -96,6 +96,30 @@ describe('runUsage — dry-run', () => { }); describe('runUsage — real path without credits (current backend)', () => { + // Confirms `usage` never sends X-CLI-Command — it must stay a plain, + // untagged /me call (only `runInit`'s configure-validate step and + // `test run --target-url`'s v3Enabled probe tag this header). + it('sends no X-CLI-Command header on its GET /me call', async () => { + writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const sentHeaders: Array | undefined> = []; + const capturingFetch = vi.fn( + async (_url: string, init: { headers?: Record }) => { + sentHeaders.push(init?.headers); + return new Response(JSON.stringify(meWithoutCredits), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }, + ) as unknown as UsageDeps['fetchImpl']; + await runUsage( + { profile: 'default', output: 'text', debug: false }, + { ...deps, credentialsPath, fetchImpl: capturingFetch }, + ); + expect(sentHeaders).toHaveLength(1); + expect(sentHeaders[0]?.['x-cli-command']).toBeUndefined(); + }); + it('returns the /me response and emits a note about missing balance', async () => { writeProfile('default', { apiKey: 'sk-user-abc' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); diff --git a/src/lib/gh-output.test.ts b/src/lib/gh-output.test.ts index 759ecaa..aefcc14 100644 --- a/src/lib/gh-output.test.ts +++ b/src/lib/gh-output.test.ts @@ -12,6 +12,7 @@ import { emitGithubOutputs, renderJobSummaryMarkdown, summarizeAcceptedPayload, + summarizeSingleRun, type CiSummary, } from './gh-output.js'; @@ -57,6 +58,85 @@ describe('summarizeAcceptedPayload', () => { expect(mixed.total).toBe(1); expect(mixed.runs[0]).toMatchObject({ testId: 'test_ok', status: 'passed' }); }); + + it('folds deferred/conflicts/notFound into non-passed rows — a partial batch is not green', () => { + const summary = summarizeAcceptedPayload( + JSON.stringify({ + accepted: [{ testId: 'test_a', runId: 'run_a', status: 'passed' }], + deferred: [{ testId: 'test_d' }], + conflicts: [{ testId: 'test_c', currentRunId: 'run_x' }], + notFound: ['test_nf'], + }), + ); + // 1 accepted-passed + 3 not-dispatched → NOT reported as "1/1 passed". + expect(summary).toMatchObject({ total: 4, passed: 1, timedOut: 0 }); + expect(summary.failed).toBe(3); + expect(summary.runs.map(r => r.status)).toEqual([ + 'passed', + 'deferred', + 'conflict', + 'not_found', + ]); + // conflict carries the in-flight runId; each incomplete row explains itself. + expect(summary.runs[2]).toMatchObject({ testId: 'test_c', runId: 'run_x', status: 'conflict' }); + expect(summary.runs[1]!.error).toContain('deferred'); + expect(summary.runs[3]).toMatchObject({ testId: 'test_nf', status: 'not_found' }); + }); + + it('an all-passed batch with empty buckets is unchanged (backward compatible)', () => { + const summary = summarizeAcceptedPayload( + JSON.stringify({ + accepted: [{ testId: 'test_a', runId: 'run_a', status: 'passed' }], + deferred: [], + conflicts: [], + notFound: [], + }), + ); + expect(summary).toMatchObject({ total: 1, passed: 1, failed: 0, timedOut: 0 }); + }); +}); + +describe('summarizeSingleRun', () => { + it('reduces a passed single run to a one-row summary', () => { + const summary = summarizeSingleRun({ + testId: 'test_a', + runId: 'run_a', + status: 'passed', + dashboardUrl: 'https://portal.example.com/a', + error: null, + }); + expect(summary).toMatchObject({ total: 1, passed: 1, failed: 0, timedOut: 0 }); + expect(summary.runs).toHaveLength(1); + expect(summary.runs[0]).toMatchObject({ + testId: 'test_a', + runId: 'run_a', + status: 'passed', + dashboardUrl: 'https://portal.example.com/a', + }); + expect(summary.runs[0]!.error).toBeUndefined(); + }); + + it('carries the raw error string (RunResponse.error is a string, not {message})', () => { + const summary = summarizeSingleRun({ + testId: 'test_b', + runId: 'run_b', + status: 'failed', + dashboardUrl: null, + error: 'assertion failed at step 2', + }); + expect(summary).toMatchObject({ total: 1, passed: 0, failed: 1, timedOut: 0 }); + expect(summary.runs[0]).toMatchObject({ testId: 'test_b', status: 'failed' }); + expect(summary.runs[0]!.error).toBe('assertion failed at step 2'); + expect(summary.runs[0]!.dashboardUrl).toBeUndefined(); + }); + + it('omits empty error / null dashboardUrl and defaults a missing status', () => { + const summary = summarizeSingleRun({ testId: 'test_c', runId: 'run_c', error: '' }); + expect(summary.runs[0]).toMatchObject({ testId: 'test_c', status: 'unknown' }); + expect(summary.runs[0]!.error).toBeUndefined(); + // a non-passed, non-timeout status counts as failed + expect(summary).toMatchObject({ total: 1, passed: 0, failed: 1 }); + }); }); describe('renderJobSummaryMarkdown', () => { @@ -66,6 +146,31 @@ describe('renderJobSummaryMarkdown', () => { expect(md).toContain('| test_a | passed | [dashboard](https://portal.example.com/a) |'); expect(md).toContain('| test_c | timeout | run_c |'); }); + + it('escapes cell content so a pipe / newline / paren cannot break or inject table rows', () => { + const md = renderJobSummaryMarkdown({ + total: 1, + passed: 0, + failed: 1, + timedOut: 0, + runs: [ + { + testId: 'evil|id\n✅ fake passed', + status: 'failed', + dashboardUrl: 'https://x.example.com/a(b)c', + }, + ], + }); + // The whole run renders as exactly ONE table row (no injected lines). + const rowLines = md.split('\n').filter(l => l.startsWith('| ') && !l.startsWith('| ---')); + // header row + the single data row + expect(rowLines).toHaveLength(2); + const dataRow = rowLines[1]!; + expect(dataRow).not.toContain('\n'); + expect(dataRow).toContain('evil\\|id'); // pipe escaped + expect(dataRow).not.toContain('✅ fake passed\n'); // newline neutralized to a space + expect(dataRow).toContain('%28b%29'); // parens in the URL percent-encoded + }); }); describe('emitGithubOutputs', () => { @@ -132,6 +237,37 @@ describe('emitGithubOutputs', () => { expect(forced.appended).toHaveLength(0); }); + it('escapes raw errors and ids so a run error cannot inject a second workflow command', () => { + const { stdout, sinks } = makeSinks(); + const malicious: CiSummary = { + total: 1, + passed: 0, + failed: 1, + timedOut: 0, + runs: [ + { + testId: 'evil:id,x', + runId: 'run_x', + status: 'failed', + error: 'line1\n::add-mask::supersecret\n::stop-commands::abc', + }, + ], + }; + emitGithubOutputs(malicious, { GITHUB_ACTIONS: 'true' }, sinks); + // One annotate() call → one array element; the invariant that neutralizes + // the injection is that the string carries NO raw newline (so the embedded + // ::add-mask:: / ::stop-commands:: sit mid-line, where Actions does not + // parse them as commands) — the newline is percent-encoded instead. + expect(stdout).toHaveLength(1); + const line = stdout[0]!; + expect(line.startsWith('::error')).toBe(true); + expect(line).not.toContain('\n'); + expect(line).not.toContain('\r'); + expect((line.match(/%0A/g) ?? []).length).toBe(2); // both newlines encoded + // testId ':' and ',' are property-escaped in the title. + expect(line).toContain('title=TestSprite evil%3Aid%2Cx::'); + }); + it('a dedicated annotations sink diverts workflow commands off the primary stdout', () => { const { stdout, sinks } = makeSinks(); const diverted: string[] = []; diff --git a/src/lib/gh-output.ts b/src/lib/gh-output.ts index ea9b379..afeddf2 100644 --- a/src/lib/gh-output.ts +++ b/src/lib/gh-output.ts @@ -1,8 +1,9 @@ /** - * CI-native output layer for the batch run path (issue #99, reshaped from - * the withdrawn top-level `ci` command per the #264 review). + * CI-native output layer for the run path (issue #99, reshaped from + * the withdrawn top-level `ci` command per the #264 review). Covers both a + * single `test run --wait` and the `test run --all --wait` batch. * - * `test run --all --wait` presents its result in the formats CI consumes: + * A `--wait` run presents its result in the formats CI consumes: * (a) a stable machine summary `{total, passed, failed, timedOut, runs[]}` * written to `--summary-file ` when requested, * (b) a Markdown results table appended to `$GITHUB_STEP_SUMMARY` when @@ -31,11 +32,41 @@ export interface CiSummary { runs: CiRunRow[]; } +/** + * Reduce a non-dispatched bucket (`deferred` / `conflicts` / `notFound`) into + * CI rows. Items are either bare testId strings (`notFound`) or + * `{ testId, currentRunId? }` objects; both shapes are handled. `note` becomes + * the row's error text so the annotation explains why the item did not run. + */ +function bucketRows(bucket: unknown, status: string, note: string): CiRunRow[] { + if (!Array.isArray(bucket)) return []; + return bucket.map(item => { + const rec = + item !== null && typeof item === 'object' ? (item as Record) : undefined; + const testId = + typeof item === 'string' ? item : typeof rec?.testId === 'string' ? rec.testId : ''; + const currentRunId = typeof rec?.currentRunId === 'string' ? rec.currentRunId : undefined; + return { + testId, + status, + ...(currentRunId ? { runId: currentRunId } : {}), + error: note, + }; + }); +} + /** * Reduce the batch command's JSON payload into the CI summary. The parse is * defensive: it reads the same `accepted[]` rows the automation contract * documents, and anything unparseable (dry-run envelope, partial output * after a timeout) reduces to an empty run list rather than a crash. + * + * Non-dispatched work (`deferred` / `conflicts` / `notFound`) is folded in as + * non-passed rows: those buckets already force a non-zero exit (deferred / + * timeout → 7, all-conflict → 6) but were previously absent from the summary, + * so a partial batch like `1 accepted passed + 1 deferred` read as "1/1 passed" + * with no annotation. (`skippedFrontend` / `skippedIntegration` are NOT folded + * in — they exit 0 and are the Action layer's allow-partial concern.) */ export function summarizeAcceptedPayload(capturedJson: string): CiSummary { let parsed: unknown; @@ -47,9 +78,13 @@ export function summarizeAcceptedPayload(capturedJson: string): CiSummary { } // `JSON.parse('null')` and non-object payloads are valid JSON but carry no // batch envelope — treat them like unparseable input instead of crashing. - const payload: { accepted?: unknown } = - parsed !== null && typeof parsed === 'object' ? (parsed as { accepted?: unknown }) : {}; - const rows: CiRunRow[] = Array.isArray(payload.accepted) + const payload: { + accepted?: unknown; + deferred?: unknown; + conflicts?: unknown; + notFound?: unknown; + } = parsed !== null && typeof parsed === 'object' ? (parsed as Record) : {}; + const acceptedRows: CiRunRow[] = Array.isArray(payload.accepted) ? payload.accepted .filter( (entry): entry is Record => entry !== null && typeof entry === 'object', @@ -68,12 +103,68 @@ export function summarizeAcceptedPayload(capturedJson: string): CiSummary { }; }) : []; + const rows: CiRunRow[] = [ + ...acceptedRows, + ...bucketRows(payload.deferred, 'deferred', 'rate-deferred (not dispatched)'), + ...bucketRows(payload.conflicts, 'conflict', 'already in flight (not dispatched)'), + ...bucketRows(payload.notFound, 'not_found', 'no replayable run (not dispatched)'), + ]; const passed = rows.filter(row => row.status === 'passed').length; const timedOut = rows.filter(row => row.status === 'timeout').length; const failed = rows.length - passed - timedOut; return { total: rows.length, passed, failed, timedOut, runs: rows }; } +/** + * Reduce a single `test run --wait` result into the same CI summary + * shape as the batch path, so `--gh-output` / `--summary-file` behave + * identically for a one-test CI job. Unlike the batch envelope, a single + * RunResponse carries `error` as a raw string (not `{ message }`) and + * `dashboardUrl` as `string | null`; both are normalized here. + */ +export function summarizeSingleRun(run: { + testId?: string; + runId?: string; + status?: string; + dashboardUrl?: string | null; + error?: string | null; +}): CiSummary { + const row: CiRunRow = { + testId: String(run.testId ?? ''), + ...(typeof run.runId === 'string' ? { runId: run.runId } : {}), + status: String(run.status ?? 'unknown'), + ...(typeof run.dashboardUrl === 'string' ? { dashboardUrl: run.dashboardUrl } : {}), + ...(typeof run.error === 'string' && run.error.length > 0 ? { error: run.error } : {}), + }; + const passed = row.status === 'passed' ? 1 : 0; + const timedOut = row.status === 'timeout' ? 1 : 0; + const failed = 1 - passed - timedOut; + return { total: 1, passed, failed, timedOut, runs: [row] }; +} + +/** + * Escape a value for a Markdown table cell. A raw `|` would break the column + * layout and a CR/LF would inject extra Markdown lines (rows are newline-joined) + * — the same injection class the annotation escaping guards against, on the + * step-summary surface. + */ +function escapeTableCell(value: string): string { + return value.replace(/\|/g, '\\|').replace(/[\r\n]+/g, ' '); +} + +/** + * Escape a URL for a Markdown `[text](url)` link: a literal `)` closes the link + * early and whitespace / `|` / CR-LF break the link or the surrounding cell. + */ +function escapeMarkdownUrl(url: string): string { + return url + .replace(/[\r\n]+/g, '') + .replace(/ /g, '%20') + .replace(/\(/g, '%28') + .replace(/\)/g, '%29') + .replace(/\|/g, '%7C'); +} + /** Markdown table for the GitHub job summary. */ export function renderJobSummaryMarkdown(summary: CiSummary): string { return [ @@ -83,16 +174,36 @@ export function renderJobSummaryMarkdown(summary: CiSummary): string { '', '| Test | Status | Run |', '| --- | --- | --- |', - ...summary.runs.map( - row => - `| ${row.testId} | ${row.status} | ${ - row.dashboardUrl ? `[dashboard](${row.dashboardUrl})` : (row.runId ?? '') - } |`, - ), + ...summary.runs.map(row => { + const run = row.dashboardUrl + ? `[dashboard](${escapeMarkdownUrl(row.dashboardUrl)})` + : escapeTableCell(row.runId ?? ''); + return `| ${escapeTableCell(row.testId)} | ${escapeTableCell(row.status)} | ${run} |`; + }), '', ].join('\n'); } +/** + * Escape a value destined for the DATA half of a workflow command (the text + * after `::`). Per GitHub's rules, `%`, CR and LF are percent-encoded so a raw + * multiline run error can never introduce a newline that starts a second + * `::command::` line in the Actions output stream (`%` first, so the encodings + * we add are not themselves re-encoded). + */ +function escapeCommandData(value: string): string { + return value.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +} + +/** + * Escape a value destined for a command PROPERTY (e.g. `title=...`). Beyond the + * data rules, `:` and `,` are encoded so the value cannot terminate the + * property list or the command header. + */ +function escapeCommandProperty(value: string): string { + return escapeCommandData(value).replace(/:/g, '%3A').replace(/,/g, '%2C'); +} + /** * Emit the GitHub-native surfaces. Self-gating on the standard env vars: * `$GITHUB_STEP_SUMMARY` (a file path Actions provides) receives the Markdown @@ -133,7 +244,11 @@ export function emitGithubOutputs( if (row.status === 'passed') continue; const detail = row.error !== undefined ? ` ${row.error}` : ''; const link = row.dashboardUrl !== undefined ? ` ${row.dashboardUrl}` : ''; - annotate(`::error title=TestSprite ${row.testId}::status=${row.status}${detail}${link}`); + // Escape both halves: a raw multiline run error (or a testId) must not be + // able to smuggle a second workflow command into the Actions stream. + const title = `TestSprite ${escapeCommandProperty(row.testId)}`; + const message = escapeCommandData(`status=${row.status}${detail}${link}`); + annotate(`::error title=${title}::${message}`); } } } diff --git a/src/lib/http.ts b/src/lib/http.ts index 65b102a..1af1f51 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -558,8 +558,18 @@ export class HttpClient { // in-flight fetch immediately (reason: InterruptError) instead of // letting a long-poll drain its window before the interrupt surfaces. if (this.shutdownSignal != null) composedSignals.push(this.shutdownSignal); - const effectiveSignal = - composedSignals.length > 1 ? AbortSignal.any(composedSignals) : timeoutSignal; + // Composed via `composeAbortSignals` (manual listeners), NOT the native + // `AbortSignal.any` — see that function's doc comment. `this.shutdownSignal` + // is process-lifetime (`globalShutdown.signal`, `client-factory.ts`) and is + // pushed into `composedSignals` on essentially every request, so + // `AbortSignal.any` here was measured to register ~330K dependent-signal + // trackings against that one long-lived signal over a single request-heavy + // test file — the same `FinalizationRegistry` backlog mechanism already + // fixed once in `poll.ts`, just recreated per-request instead of + // per-poll-iteration. `cleanupAbortComposition` MUST be called once the + // request settles (below, alongside `requestTimeout.clear()`). + const { signal: effectiveSignal, cleanup: cleanupAbortComposition } = + composeAbortSignals(composedSignals); try { try { @@ -797,6 +807,11 @@ export class HttpClient { await this.sleepBeforeRetry(decision.delayMs); } finally { requestTimeout.clear(); + // Remove the manually-added abort listeners now, synchronously — + // see `composeAbortSignals`. Runs on every path out of the try + // (return, throw, or `continue` into the next attempt), so nothing + // outlives the request it was created for. + cleanupAbortComposition(); } } } @@ -905,6 +920,68 @@ interface RequestTimeoutHandle { clear: () => void; } +interface AbortComposition { + signal: AbortSignal; + cleanup: () => void; +} + +/** + * Compose multiple `AbortSignal`s into one — abort on whichever fires first — + * WITHOUT using the native `AbortSignal.any`. + * + * `AbortSignal.any` builds a persistent "dependent signal" relationship: the + * signal it returns is tracked (via a `WeakRef` + `FinalizationRegistry`) + * against EVERY source signal passed in, including any long-lived one. Every + * outgoing request here composes `this.shutdownSignal` — process-lifetime + * (`globalShutdown.signal`, wired by `client-factory.ts` on essentially every + * command invocation) — into `composedSignals`, so each request added one + * more tracked dependent against that single long-lived signal, reclaimed + * only whenever V8 next got around to running ITS `FinalizationRegistry` + * cleanup pass. Under a request-heavy path (retries, a wide closure/batch + * fan-out, RATE_LIMITED backoff) this reproduces — per HTTP request instead + * of per poll-iteration — the exact backlog mechanism already fixed once in + * `poll.ts` (`JSFinalizationRegistry::Cleanup` → `KeepDuringJob` → + * `OrderedHashSet::Add` pegging the CPU once the deferred backlog is finally + * walked). Measured directly: the RATE_LIMITED closure-fanout scenario in + * `test.rerun.spec.ts` alone drives ~330K `AbortSignal.any` calls against the + * one process-lifetime `globalShutdown.signal` over the life of that single + * test file. + * + * This performs the identical "first one wins, propagate its `.reason`" + * composition with a plain `AbortController` and ordinary + * `addEventListener`/`removeEventListener` — no dependent-signal tracking, no + * `WeakRef`, no GC-deferred cleanup. The caller MUST invoke the returned + * `cleanup()` once the request settles (success, error, or retry) so the + * listeners are removed immediately rather than left for a cleanup pass that, + * for this composition, no longer exists. + */ +function composeAbortSignals(signals: readonly AbortSignal[]): AbortComposition { + if (signals.length <= 1) { + return { signal: signals[0]!, cleanup: () => {} }; + } + // Mirror AbortSignal.any's already-aborted short-circuit: if a source is + // already aborted at composition time, propagate its reason immediately — + // no listeners are ever added, so cleanup is a no-op. + const alreadyAborted = signals.find(s => s.aborted); + const controller = new AbortController(); + if (alreadyAborted) { + controller.abort(alreadyAborted.reason); + return { signal: controller.signal, cleanup: () => {} }; + } + const removers: Array<() => void> = []; + for (const source of signals) { + const onAbort = (): void => controller.abort(source.reason); + source.addEventListener('abort', onAbort, { once: true }); + removers.push(() => source.removeEventListener('abort', onAbort)); + } + return { + signal: controller.signal, + cleanup: () => { + for (const remove of removers) remove(); + }, + }; +} + function createRequestTimeout(timeoutMs: number): RequestTimeoutHandle { const controller = new AbortController(); const timer = setTimeout(() => { diff --git a/src/lib/poll.spec.ts b/src/lib/poll.spec.ts index 93889eb..09eb268 100644 --- a/src/lib/poll.spec.ts +++ b/src/lib/poll.spec.ts @@ -541,7 +541,14 @@ describe('pollRunUntilTerminal — AbortSignal + timeout enforcement', () => { expect(receivedSignals[0]).toBeInstanceOf(AbortSignal); }); - it('passes a fresh AbortSignal on each poll iteration', async () => { + it('reuses the same session AbortSignal across poll iterations (no per-iteration churn)', async () => { + // Regression guard for the AbortController/AbortSignal.any churn fix: a + // fresh controller + composed signal per iteration was pure waste (the + // abort target — deadlineMs + cushion — never changes between + // iterations), and on a `--wait` fan-out over many concurrent runIds it + // produced enough short-lived `AbortSignal.any` composites to make V8's + // FinalizationRegistry cleanup pass pathologically slow. The signal is + // now hoisted once per poll session and reused for every iteration. const receivedSignals: Array = []; const client: RunClient = { getRun: async (_runId, opts) => { @@ -555,8 +562,8 @@ describe('pollRunUntilTerminal — AbortSignal + timeout enforcement', () => { sleep: instantSleep, }); expect(receivedSignals).toHaveLength(2); - // Each iteration gets its own controller → distinct signal objects - expect(receivedSignals[0]).not.toBe(receivedSignals[1]); + expect(receivedSignals[0]).toBeInstanceOf(AbortSignal); + expect(receivedSignals[0]).toBe(receivedSignals[1]); }); it('surfaces TimeoutError when fetch resolves as AbortError (hung fetch past deadline)', async () => { diff --git a/src/lib/poll.ts b/src/lib/poll.ts index 94b3ae4..fdd74ed 100644 --- a/src/lib/poll.ts +++ b/src/lib/poll.ts @@ -136,7 +136,7 @@ async function pollLoop( runId: string, options: PollOptions, ): Promise { - const { timeoutSeconds, onTick, onTransition, resolveAlternate } = options; + const { timeoutSeconds, resolveAlternate } = options; const shutdownSignal = options.shutdown?.signal; const rawSleep = options.sleep ?? defaultSleep; // Every sleep site (retryAfterSeconds, backoff schedule, not_yet_visible, @@ -147,6 +147,89 @@ async function pollLoop( const startMs = Date.now(); const deadlineMs = startMs + timeoutSeconds * 1000; + // Hoisted ONCE per poll *session* rather than re-minted every iteration. + // The abort target here — deadlineMs + a transport cushion — is a fixed + // absolute instant that does not change from one iteration to the next + // (remainingMs shrinks, but remainingMs + TRANSPORT_CUSHION_MS measured + // from "now" always lands on the same deadlineMs + cushion point). Minting + // a fresh AbortController + setTimeout + `AbortSignal.any` composite every + // iteration was therefore pure churn with no behavioral purpose: on a + // `--wait` fan-out over many concurrent runIds, each polling for many + // iterations, this created thousands of composite signals per invocation. + // Every `AbortSignal.any` call registers an internal dependency listener + // on the long-lived `shutdownSignal`, tracked via a WeakRef + + // FinalizationRegistry so the listener can be reclaimed once the derived + // signal is garbage-collected without leaking on the long-lived source — + // but under a high allocation rate with no macrotask yield between + // iterations (`TestDeps.sleep` in tests resolves via a bare microtask), + // V8 can fall behind on running that reclamation incrementally, and the + // eventual cleanup pass then has to walk a large backlog in one go + // (`JSFinalizationRegistry::Cleanup` → `KeepDuringJob` → + // `OrderedHashSet::Add`), pegging the CPU for minutes. One controller, + // one timer, and one composed signal — created once, reused for every + // iteration and every retry, cleared once when the session ends — + // removes that per-iteration scaling factor entirely; the deadline + // semantics are unchanged, since the abort target was already the same + // absolute instant on every iteration. + const TRANSPORT_CUSHION_MS = 2000; + const deadlineController = new AbortController(); + const deadlineTimer = setTimeout( + () => deadlineController.abort(), + deadlineMs - startMs + TRANSPORT_CUSHION_MS, + ); + const sessionSignal = + shutdownSignal != null + ? AbortSignal.any([deadlineController.signal, shutdownSignal]) + : deadlineController.signal; + + // Same hoisting rationale for the `resolveAlternate` abort target + // (deadlineMs, no cushion — also a fixed absolute instant). Allocated + // lazily so callers that never pass `resolveAlternate` pay nothing for it. + let altAbort: AbortController | undefined; + let altTimer: ReturnType | undefined; + let altSignal: AbortSignal | undefined; + if (resolveAlternate) { + altAbort = new AbortController(); + const controller = altAbort; + altTimer = setTimeout(() => controller.abort(), deadlineMs - startMs); + altSignal = + shutdownSignal != null ? AbortSignal.any([altAbort.signal, shutdownSignal]) : altAbort.signal; + } + + try { + return await pollIterations(client, runId, options, { + startMs, + deadlineMs, + sessionSignal, + altSignal, + sleep, + }); + } finally { + clearTimeout(deadlineTimer); + if (altTimer !== undefined) clearTimeout(altTimer); + } +} + +interface PollLoopContext { + startMs: number; + deadlineMs: number; + /** Hoisted per-session signal for the run-row GET (deadline + cushion, composed with shutdown). */ + sessionSignal: AbortSignal; + /** Hoisted per-session signal for `resolveAlternate` lookups (deadline, no cushion), if applicable. */ + altSignal: AbortSignal | undefined; + sleep: (ms: number) => Promise; +} + +async function pollIterations( + client: RunClient, + runId: string, + options: PollOptions, + ctx: PollLoopContext, +): Promise { + const { timeoutSeconds, onTick, onTransition, resolveAlternate } = options; + const shutdownSignal = options.shutdown?.signal; + const { startMs, deadlineMs, sessionSignal, altSignal, sleep } = ctx; + // Track whether the server supports ?waitSeconds. Start optimistic. let useBackoff = false; let backoffIndex = 0; @@ -166,43 +249,25 @@ async function pollLoop( const remainingMs = deadlineMs - now; const remainingSeconds = Math.ceil(remainingMs / 1000); - // Mint a per-iteration AbortController. The signal fires at the remaining - // deadline plus a small transport cushion (2 s) so a hung fetch does not - // block the CLI past --timeout. - const TRANSPORT_CUSHION_MS = 2000; - const abortController = new AbortController(); - const abortTimer = setTimeout(() => { - abortController.abort(); - }, remainingMs + TRANSPORT_CUSHION_MS); - // Compose the interrupt into the per-iteration signal: a `--wait` can sit - // inside one <=25s long-poll fetch (and the auto-raised per-request - // timeout means even longer for slow backends) — checking the flag - // between iterations is not enough, the in-flight fetch must abort. - const iterationSignal = - shutdownSignal != null - ? AbortSignal.any([abortController.signal, shutdownSignal]) - : abortController.signal; - let run: RunResponse; try { if (useBackoff) { - run = await client.getRun(runId, { signal: iterationSignal }); + run = await client.getRun(runId, { signal: sessionSignal }); } else { const waitSeconds = Math.min(remainingSeconds, LONG_POLL_WAIT_SECONDS); - run = await client.getRun(runId, { waitSeconds, signal: iterationSignal }); + run = await client.getRun(runId, { waitSeconds, signal: sessionSignal }); } // Successful GET resets the consecutive-error counter. consecutiveErrors = 0; notYetVisibleRetries = 0; } catch (err) { - clearTimeout(abortTimer); // Interrupt classification precedes the timeout mapping: the composed // signal makes the fetch reject on Ctrl-C, and that abort must surface // as the InterruptError — not as a spurious TimeoutError. if (err instanceof InterruptError) throw err; if (shutdownSignal?.aborted) throw shutdownSignal.reason; - // An AbortError from our per-iteration controller means the deadline - // passed while the fetch was in flight — surface as TimeoutError. + // An AbortError from the session controller means the deadline passed + // while the fetch was in flight — surface as TimeoutError. if (isAbortError(err)) { throw new TimeoutError(runId, timeoutSeconds); } @@ -264,9 +329,6 @@ async function pollLoop( throw err; } - // fetch completed — cancel the per-iteration abort timer. - clearTimeout(abortTimer); - const elapsedMs = Date.now() - startMs; onTick?.(run, elapsedMs); @@ -285,21 +347,13 @@ async function pollLoop( if (altRemainingMs <= 0) { throw new TimeoutError(runId, timeoutSeconds); } - const altAbort = new AbortController(); - const altTimer = setTimeout(() => altAbort.abort(), altRemainingMs); // The alternate lookup aborts on interrupt too; its errors are swallowed // by the fallback (best-effort), so the loop-top interrupt check above - // surfaces the InterruptError on the next iteration. - const altSignal = - shutdownSignal != null - ? AbortSignal.any([altAbort.signal, shutdownSignal]) - : altAbort.signal; - let alternate: RunResponse | null; - try { - alternate = await resolveAlternate(run, elapsedMs, altSignal); - } finally { - clearTimeout(altTimer); - } + // surfaces the InterruptError on the next iteration. `altSignal` is the + // hoisted per-session signal (allocated once, above, whenever + // `resolveAlternate` is present) — reused across every non-terminal + // tick rather than re-composed each time. + const alternate = await resolveAlternate(run, elapsedMs, altSignal!); // Enforce the hard cap: reject a terminal alternate that only arrived // at/after the deadline, same as the run-row long-poll path below. if (Date.now() >= deadlineMs) { diff --git a/src/lib/response-schemas.ts b/src/lib/response-schemas.ts index c551048..1e5ff5c 100644 --- a/src/lib/response-schemas.ts +++ b/src/lib/response-schemas.ts @@ -109,24 +109,34 @@ export const RUN_RESPONSE_SCHEMA: v.GenericSchema = v.loos videoUrl: v.nullish(v.string(), null), stepSummary: RUN_STEP_SUMMARY_SCHEMA, retryAfterSeconds: v.optional(v.number()), - // Portal link. Newer backends DO send this (they alone know which store - // answered the read and can resolve a non-prod portal origin); older ones - // omit it and the CLI computes its own. `nullish` rather than `optional` - // deliberately: the backend omits the field when no correct link exists, but - // a `null` from any other producer must not fail validation and take down - // `test wait` — the same trap that had to be un-sprung for a null - // `targetUrl`/`codeVersion`. + // Portal link. Three-state wire contract (pinned): **absent** — an older + // backend that predates this field, and cannot have produced a V3-native/ + // unmirrored entity either (that capability and this field ship together) + // — the CLI computes its own legacy V2-shaped link. **Present + string** — + // the backend built a correct link (it alone knows which store answered + // and this environment's portal origin) — use it verbatim. **Present + + // `null`** — the backend deliberately has no correct link to offer (e.g. a + // V3-native entity with no DynamoDB mirror row for the client's V2-shaped + // guess to land on) — suppress the link entirely; a client-side guess here + // would be exactly the dead link the server declined to emit. The backend + // always includes the key going forward (typed `string | null`, never + // omitted when it has an opinion) — an earlier revision of this comment + // described the backend as omitting the key on "no correct link", which + // was the actual production defect this contract closes: the client's + // absent-branch fallback was firing on real V3-native no-link responses + // and printing the dead legacy URL this whole feature exists to remove. // // The `undefined` default (NOT `null`, unlike every field above) is load-bearing // and measured: valibot applies a default only when the key is absent, and // skips the assignment entirely when that default is `undefined` — so an // omitted field stays an ABSENT key, which is exactly what - // `withRunDashboardUrl`'s `'dashboardUrl' in run` test reads to decide - // "old backend, compute the link myself". Aligning this with the - // `nullish(..., null)` fields above would materialize the key on every - // response and silently kill that fallback. A wire `null` is preserved as - // null here (nullable passes it through untouched) and normalized at the - // consumer, not in the schema. Locked by tests in response-schemas.test.ts. + // `withRunDashboardUrl`'s `'dashboardUrl' in run` test (via the shared + // `resolveDashboardUrl` helper) reads to decide "old backend, compute the + // link myself". Aligning this with the `nullish(..., null)` fields above + // would materialize the key on every response and silently kill that + // fallback. A wire `null` is preserved as null here (nullable passes it + // through untouched) and normalized at the consumer, not in the schema. + // Locked by tests in response-schemas.test.ts. dashboardUrl: v.nullish(v.string(), undefined), // Absence means "steps not requested" and drives command branching, so no // default is applied (rule 3, optional branch). diff --git a/src/lib/v3-advisory.test.ts b/src/lib/v3-advisory.test.ts index 05eac19..7812672 100644 --- a/src/lib/v3-advisory.test.ts +++ b/src/lib/v3-advisory.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { routingLabel, V3_ROUTING_ADVISORY, emitV3RoutingAdvisory } from './v3-advisory.js'; +import { + routingLabel, + V3_ROUTING_ADVISORY, + emitV3RoutingAdvisory, + TARGET_URL_V3_ADVISORY, + emitTargetUrlV3Advisory, +} from './v3-advisory.js'; describe('routingLabel', () => { it('maps the boolean to v3 / v2', () => { @@ -28,3 +34,19 @@ describe('V3 routing advisory', () => { expect(lines).toEqual(V3_ROUTING_ADVISORY); }); }); + +describe('target-url V3 advisory (DEV-749)', () => { + it('is a single [advisory]-prefixed line naming --target-url', () => { + expect(TARGET_URL_V3_ADVISORY.startsWith('[advisory]')).toBe(true); + expect(TARGET_URL_V3_ADVISORY).toContain('--target-url'); + // Type-agnostic: never claims this is frontend-only, since the CLI + // cannot learn test type without an extra round trip at `test run` time. + expect(TARGET_URL_V3_ADVISORY).not.toContain('frontend'); + }); + + it('emitTargetUrlV3Advisory writes exactly one line to the sink', () => { + const lines: string[] = []; + emitTargetUrlV3Advisory(l => lines.push(l)); + expect(lines).toEqual([TARGET_URL_V3_ADVISORY]); + }); +}); diff --git a/src/lib/v3-advisory.ts b/src/lib/v3-advisory.ts index 262c467..2b1a5db 100644 --- a/src/lib/v3-advisory.ts +++ b/src/lib/v3-advisory.ts @@ -31,3 +31,29 @@ export const V3_ROUTING_ADVISORY: string[] = [ export function emitV3RoutingAdvisory(stderr: (line: string) => void): void { for (const line of V3_ROUTING_ADVISORY) stderr(line); } + +/** + * Point-of-use advisory for `test run --target-url` on a V3-routed caller + * (DEV-749). `V3_ROUTING_ADVISORY` above already names this gap once, in + * the account-level summary `auth status`/`doctor` print — this is the + * SAME gap surfaced at the moment the caller actually hits it, matching + * the existing backend-test `--target-url` advisory in `runCreate` + * (`commands/test.ts`): unconditional across every `--output` mode. That + * advisory's family is "a flag the caller just passed has a structural + * consequence" — `--output json` is precisely the unattended/CI case that + * needs the warning most, not the case to withhold it from (unlike the + * routing-advisory family above, which a JSON caller can skip by reading + * `v3Enabled` directly off that command's own structured output — `test + * run`'s JSON output carries no such field). Deliberately type-agnostic + * (no "on frontend runs" claim): the CLI cannot learn a test's type at + * `test run ` time without an extra round trip, and the + * override is equally inert on the V3 backend-run path. + */ +export const TARGET_URL_V3_ADVISORY = + "[advisory] --target-url is not applied on the V3 execution path; the run uses the test's " + + 'configured environment instead.'; + +/** Write the target-url advisory to a stderr sink. */ +export function emitTargetUrlV3Advisory(stderr: (line: string) => void): void { + stderr(TARGET_URL_V3_ADVISORY); +} diff --git a/src/version.ts b/src/version.ts index 80229a2..0daf75e 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.5.0'; +export const VERSION = '0.6.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 1e4802c..0a26168 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -470,7 +470,7 @@ Options: Plan file format (--plan-from ) — minimal valid example: { - "$schema": "https://raw.githubusercontent.com/TestSprite/testsprite-cli/v0.5.0/schemas/plan.schema.json", + "$schema": "https://raw.githubusercontent.com/TestSprite/testsprite-cli/v0.6.0/schemas/plan.schema.json", "projectId": "prj_abc123", "type": "frontend", "name": "Login rejects an empty password", @@ -685,6 +685,13 @@ Options: --report-file output path for --report (atomic write) --report-suite-name optional JUnit override (default: testsprite:) + --gh-output with batch --wait: emit GitHub-native output + (::error:: annotations per non-passed run; + job-summary table when $GITHUB_STEP_SUMMARY is + set). Auto-enabled when GITHUB_ACTIONS=true + --summary-file with batch --wait: also write the reduced machine + summary JSON {total, passed, failed, timedOut, + runs[]} to this file -h, --help display help for command Notes: @@ -791,13 +798,14 @@ Options: --report-file output path for --report (atomic write) --report-suite-name optional JUnit override (default: testsprite:) - --gh-output with --all --wait: emit GitHub-native output - (::error:: annotations per non-passed run; - job-summary table when $GITHUB_STEP_SUMMARY is - set). Auto-enabled when GITHUB_ACTIONS=true - --summary-file with --all --wait: also write the reduced machine - summary JSON {total, passed, failed, timedOut, - runs[]} to this file + --gh-output with --wait (single test or --all): emit + GitHub-native output (::error:: annotations per + non-passed run; job-summary table when + $GITHUB_STEP_SUMMARY is set). Auto-enabled when + GITHUB_ACTIONS=true + --summary-file with --wait (single test or --all): also write + the reduced machine summary JSON {total, passed, + failed, timedOut, runs[]} to this file -h, --help display help for command Dependency-aware fresh run (M4): diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index a37ea5d..10f8d79 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -376,6 +376,11 @@ beforeAll(async () => { afterAll(async () => { await new Promise(resolveClose => server.close(() => resolveClose())); + // tmpHome accumulates a real (if fake-valued) credentials file across this + // suite's `auth configure` / `setup` runs — remove it so the suite doesn't + // leave one directory behind per test run. Guarded: if `beforeAll` threw + // before assigning it, there is nothing to remove. + if (tmpHome) rmSync(tmpHome, { recursive: true, force: true }); }); interface SpawnResult { diff --git a/test/contract/p4-schema.test.ts b/test/contract/p4-schema.test.ts index ac2f5ae..c90e5f9 100644 --- a/test/contract/p4-schema.test.ts +++ b/test/contract/p4-schema.test.ts @@ -24,10 +24,10 @@ * regression escapes to dev. */ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { writeFileSync } from 'node:fs'; import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it } from 'vitest'; +import { makeTempDir } from '../helpers/tempDir.js'; import { runCodeGet, runResult, runSteps } from '../../src/commands/test.js'; import { failureContextFixture, @@ -269,10 +269,14 @@ describe('P4 schema contract — fixtures match the OpenAPI shapes', () => { // CLI's typed contract matches the wire shape." If a runner ever // silently omits or coerces a field, this fails. +// Every makeCreds() call gets its own temp dir; each is registered here and +// swept in afterEach so a failing assertion mid-test still leaves nothing behind. +const pendingCredsCleanups: Array<() => void> = []; + function makeCreds(): { credentialsPath: string } { - const dir = mkdtempSync(join(tmpdir(), 'cli-p4-contract-')); - const credentialsPath = join(dir, 'credentials'); - mkdirSync(dir, { recursive: true }); + const dir = makeTempDir('cli-p4-contract-'); + pendingCredsCleanups.push(dir.cleanup); + const credentialsPath = join(dir.path, 'credentials'); // The base URL must match what the MSW handlers serve (DEFAULT_BASE_URL // sans the /api/cli/v1 suffix that facadeBaseUrl re-appends). writeFileSync( @@ -283,6 +287,10 @@ function makeCreds(): { credentialsPath: string } { return { credentialsPath }; } +afterEach(() => { + while (pendingCredsCleanups.length > 0) pendingCredsCleanups.pop()?.(); +}); + describe('P4 schema contract — CLI runners return §6.x shapes', () => { it('runCodeGet (inline) returns a §6.3 TestCode', async () => { const { credentialsPath } = makeCreds(); diff --git a/test/contract/p5-schema.test.ts b/test/contract/p5-schema.test.ts index 6a7edb3..cf9fdc4 100644 --- a/test/contract/p5-schema.test.ts +++ b/test/contract/p5-schema.test.ts @@ -16,7 +16,7 @@ * and these validators are the third gate. */ -import { describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it } from 'vitest'; import type { CliFailureContext, CliFailureBlock, @@ -179,15 +179,19 @@ describe('P5 schema contract — FailureContext fixtures match the OpenAPI shape // ---- CLI surface contract: what runFailureGet returns ---- -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { writeFileSync } from 'node:fs'; import { join } from 'node:path'; +import { makeTempDir } from '../helpers/tempDir.js'; import { runFailureGet } from '../../src/commands/test.js'; +// Every makeCreds() call gets its own temp dir; each is registered here and +// swept in afterEach so a failing assertion mid-test still leaves nothing behind. +const pendingCredsCleanups: Array<() => void> = []; + function makeCreds(): { credentialsPath: string } { - const dir = mkdtempSync(join(tmpdir(), 'cli-p5-contract-')); - const credentialsPath = join(dir, 'credentials'); - mkdirSync(dir, { recursive: true }); + const dir = makeTempDir('cli-p5-contract-'); + pendingCredsCleanups.push(dir.cleanup); + const credentialsPath = join(dir.path, 'credentials'); writeFileSync( credentialsPath, `[default]\napi_url = https://api.testsprite.com\napi_key = sk-user-test\n`, @@ -196,6 +200,10 @@ function makeCreds(): { credentialsPath: string } { return { credentialsPath }; } +afterEach(() => { + while (pendingCredsCleanups.length > 0) pendingCredsCleanups.pop()?.(); +}); + describe('P5 schema contract — runFailureGet returns a §6.7 FailureContext', () => { it('runFailureGet returns the wire envelope for a known failing test', async () => { const { credentialsPath } = makeCreds(); diff --git a/test/helpers/tempDir.ts b/test/helpers/tempDir.ts new file mode 100644 index 0000000..09fc903 --- /dev/null +++ b/test/helpers/tempDir.ts @@ -0,0 +1,31 @@ +/** + * Shared temp-directory helper for tests that scratch files to disk — + * credentials files included. + * + * `mkdtempSync` (unlike a bare `mkdirSync`) creates the directory with + * owner-only `0700` permissions on POSIX by construction, matching the mode + * the CLI's own credential writer applies by hand + * (`mkdirSync(dirname(path), { recursive: true, mode: 0o700 })` in + * `src/lib/credentials.ts`). Pairing every `makeTempDir` with its returned + * `cleanup()` in a `finally`/`afterEach` is what keeps a test that writes a + * throwaway API key (or any other secret-shaped fixture) from leaving that + * directory behind on disk after the run — hand-rolled + * `mkdirSync`/`join(tmpdir(), ...)` call sites have no such guarantee and + * are easy to forget to clean up. + */ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +export interface TempDir { + /** Absolute path to the created directory. */ + path: string; + /** Removes the directory (recursive, force) — safe to call even if the directory was already removed. */ + cleanup: () => void; +} + +/** Create a fresh, uniquely-named temp directory under the OS tmpdir. */ +export function makeTempDir(prefix: string): TempDir { + const path = mkdtempSync(join(tmpdir(), prefix)); + return { path, cleanup: () => rmSync(path, { recursive: true, force: true }) }; +} From f260e931fcf30b8ea473fe592b5af196b1bf55ac Mon Sep 17 00:00:00 2001 From: nopp Date: Thu, 13 Aug 2026 15:54:39 +0700 Subject: [PATCH 113/117] fix(project): guard --password-file reads on create/update (#79) (#302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `project create` and `project update` resolved `--password-file` with a bare `readFileSync(path, 'utf8').trim()`. A path typo — the expected failure mode for a hand-typed flag — escaped as an unhandled Node exception: - exit `1` (generic) instead of `5` (validation) - an `--output json` payload whose `error` is a bare string, not the `{ code, message, nextAction }` envelope the rest of the CLI emits, so anything parsing `--output json` breaks - the absolute path and errno leaked to stderr Add `readSecretFileGuarded` in `src/lib/secret-file.ts`, mirroring `readCodeFileGuarded` in `src/commands/test.ts`, and route both call sites through it. Missing paths, permission failures, and directories now produce the standard VALIDATION_ERROR envelope naming the flag. The helper takes the flag name so the remaining unguarded file flags (`--credential-file`, `--client-secret-file`, `--refresh-token-file`, and `project auto-auth --password-file`) can adopt it under #282 without rework — those sites are deliberately left untouched here to avoid colliding with that issue's in-progress work. The payload cap from `readCodeFileGuarded` is not carried over: secrets are small, and a size ceiling would be a behaviour change on a shipped flag rather than part of fixing the crash. Dry-run behaviour is unchanged — both paths already return before password resolution, and the existing P7 coverage still passes. --- src/commands/project.test.ts | 116 ++++++++++++++++++++++++++++++++++ src/commands/project.ts | 5 +- src/lib/secret-file.test.ts | 118 +++++++++++++++++++++++++++++++++++ src/lib/secret-file.ts | 87 ++++++++++++++++++++++++++ 4 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 src/lib/secret-file.test.ts create mode 100644 src/lib/secret-file.ts diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 60a1671..07fd321 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -1583,6 +1583,122 @@ describe('runUpdate', () => { }); }); +describe('#79 — an unreadable --password-file is a validation error, not a crash', () => { + const missing = join(tmpdir(), 'testsprite-issue-79-absent-password-file'); + + it('runCreate rejects a missing file with VALIDATION_ERROR (exit 5) before the network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'backend', + name: 'Guarded', + passwordFile: missing, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('runUpdate rejects a missing file with VALIDATION_ERROR (exit 5) before the network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_guarded', + passwordFile: missing, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('names the flag in nextAction instead of leaking a raw ENOENT', async () => { + const { credentialsPath } = makeCreds(); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'backend', + name: 'Guarded', + passwordFile: missing, + }, + { + credentialsPath, + fetchImpl: (async () => { + throw new Error('should not hit network'); + }) as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ + nextAction: expect.stringContaining('--password-file') as unknown as string, + }); + }); + + it('still reads a password file that exists', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p79-')); + const passwordFile = join(dir, 'pw.txt'); + writeFileSync(passwordFile, 'from-file\n'); + + const sentBodies: unknown[] = []; + const fetchImpl = (async (_input: Parameters[0], init: RequestInit = {}) => { + if (init.body) sentBodies.push(JSON.parse(init.body as string) as unknown); + return new Response(JSON.stringify({ ...PROJECT_FIXTURE, id: 'proj_pw' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'backend', + name: 'Guarded', + passwordFile, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect((sentBodies[0] as Record).password).toBe('from-file'); + }); +}); + describe('runDelete', () => { it('refuses without --confirm and never hits the network (exit 5)', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/project.ts b/src/commands/project.ts index fd0c15b..11f31f2 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -11,6 +11,7 @@ import { ApiError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; +import { readSecretFileGuarded } from '../lib/secret-file.js'; import { assertNotLocal } from '../lib/target-url.js'; import { renderTextTable, resolveTextColumns, type TextTableColumn } from '../lib/text-table.js'; import { assertIdempotencyKey } from '../lib/validate.js'; @@ -276,7 +277,7 @@ export async function runCreate( // Resolve password: flag > file > none let password = opts.password; if (password === undefined && opts.passwordFile !== undefined) { - password = readFileSync(opts.passwordFile, 'utf8').trim(); + password = readSecretFileGuarded('password-file', opts.passwordFile); } const idempotencyKey = opts.idempotencyKey ?? `cli-proj-create-${randomUUID()}`; @@ -463,7 +464,7 @@ export async function runUpdate( // filesystem, even when --password-file is present. let password = opts.password; if (password === undefined && opts.passwordFile !== undefined) { - password = readFileSync(opts.passwordFile, 'utf8').trim(); + password = readSecretFileGuarded('password-file', opts.passwordFile); } const idempotencyKey = opts.idempotencyKey ?? `cli-proj-update-${randomUUID()}`; diff --git a/src/lib/secret-file.test.ts b/src/lib/secret-file.test.ts new file mode 100644 index 0000000..f22a518 --- /dev/null +++ b/src/lib/secret-file.test.ts @@ -0,0 +1,118 @@ +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { ApiError } from './errors.js'; +import { readSecretFileGuarded } from './secret-file.js'; + +let tmpRoot: string; +const originalCwd = process.cwd(); + +beforeEach(() => { + tmpRoot = mkdtempSync(join(tmpdir(), 'testsprite-secret-file-')); +}); + +afterEach(() => { + // mkdtempSync directory is small and short-lived; OS cleans it up. + process.chdir(originalCwd); +}); + +describe('readSecretFileGuarded', () => { + it('returns the file contents', () => { + const path = join(tmpRoot, 'pw.txt'); + writeFileSync(path, 'hunter2'); + expect(readSecretFileGuarded('password-file', path)).toBe('hunter2'); + }); + + it('trims surrounding whitespace and the trailing newline', () => { + const path = join(tmpRoot, 'pw-newline.txt'); + writeFileSync(path, ' hunter2 \n'); + expect(readSecretFileGuarded('password-file', path)).toBe('hunter2'); + }); + + it('drops a leading UTF-8 BOM so PowerShell-written files still work', () => { + const path = join(tmpRoot, 'pw-bom.txt'); + writeFileSync(path, 'hunter2\n'); + expect(readSecretFileGuarded('password-file', path)).toBe('hunter2'); + }); + + it('preserves interior whitespace', () => { + const path = join(tmpRoot, 'pw-spaces.txt'); + writeFileSync(path, 'two words\n'); + expect(readSecretFileGuarded('password-file', path)).toBe('two words'); + }); + + it('resolves a relative path against the working directory', () => { + writeFileSync(join(tmpRoot, 'relative.txt'), 'from-cwd'); + process.chdir(tmpRoot); + expect(readSecretFileGuarded('password-file', 'relative.txt')).toBe('from-cwd'); + }); + + it('returns an empty string for an empty file rather than throwing', () => { + const path = join(tmpRoot, 'empty.txt'); + writeFileSync(path, ''); + expect(readSecretFileGuarded('password-file', path)).toBe(''); + }); + + describe('missing file', () => { + it('throws VALIDATION_ERROR with exit code 5', () => { + const path = join(tmpRoot, 'nope.txt'); + expect(() => readSecretFileGuarded('password-file', path)).toThrow(ApiError); + try { + readSecretFileGuarded('password-file', path); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + } + }); + + it('names the offending flag and path in nextAction', () => { + const path = join(tmpRoot, 'nope.txt'); + try { + readSecretFileGuarded('password-file', path); + expect.unreachable('should have thrown'); + } catch (err) { + const { nextAction } = err as ApiError; + expect(nextAction).toContain('--password-file'); + expect(nextAction).toContain('file does not exist'); + expect(nextAction).toContain(path); + } + }); + + it('attributes the error to whichever flag the caller names', () => { + const path = join(tmpRoot, 'nope.txt'); + try { + readSecretFileGuarded('client-secret-file', path); + expect.unreachable('should have thrown'); + } catch (err) { + expect((err as ApiError).nextAction).toContain('--client-secret-file'); + } + }); + + it('reports the path as typed, not the resolved absolute path', () => { + process.chdir(tmpRoot); + try { + readSecretFileGuarded('password-file', 'missing.txt'); + expect.unreachable('should have thrown'); + } catch (err) { + const { nextAction } = err as ApiError; + expect(nextAction).toContain('missing.txt'); + expect(nextAction).not.toContain(tmpRoot); + } + }); + }); + + describe('directory instead of a file', () => { + it('throws VALIDATION_ERROR instead of crashing with EISDIR', () => { + const path = join(tmpRoot, 'a-directory'); + mkdirSync(path); + try { + readSecretFileGuarded('password-file', path); + expect.unreachable('should have thrown'); + } catch (err) { + expect(err).toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect((err as ApiError).nextAction).toContain('not a regular file'); + } + }); + }); +}); diff --git a/src/lib/secret-file.ts b/src/lib/secret-file.ts new file mode 100644 index 0000000..b9619e6 --- /dev/null +++ b/src/lib/secret-file.ts @@ -0,0 +1,87 @@ +/** + * Guarded reader for the `--*-file` secret flags. + * + * Every one of these flags exists so a secret stays out of shell history, and + * every one of them is a path the user types by hand — so a typo is the + * expected failure, not an exceptional one. A bare + * `readFileSync(path, 'utf8').trim()` turns that typo into an unhandled Node + * exception: exit `1` instead of `5`, an `--output json` payload whose `error` + * is a bare string rather than the `{ code, message, nextAction }` envelope the + * rest of the CLI emits, and the absolute path plus errno leaked to stderr. + * + * This maps those failures onto the same typed `VALIDATION_ERROR` envelope the + * already-guarded file flags produce, mirroring `readCodeFileGuarded` in + * `src/commands/test.ts`. The payload cap is deliberately not carried over: + * secrets are small, and a size ceiling would be a behaviour change on a + * shipped flag rather than part of fixing the crash. + * + * Callers pass the flag name so the envelope names the flag the user actually + * typed — one helper serves `--password-file` today and the remaining + * credential/auto-auth file flags once they are migrated. + */ +import { readFileSync, statSync } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; +import { localValidationError } from './errors.js'; + +/** + * Read a secret from `path`, surfacing every filesystem failure as a typed + * `VALIDATION_ERROR` (exit 5) attributed to `flag`. + * + * The returned value is trimmed, matching what the unguarded call sites did. + * Trimming also drops a leading UTF-8 BOM: `U+FEFF` is ECMAScript whitespace, + * so a file written by PowerShell 5.1's default `Set-Content -Encoding utf8` + * no longer smuggles an invisible character into the secret. + * + * @param flag - Flag name without the leading dashes, e.g. `'password-file'`. + * @param path - Path as supplied by the user; may be relative. + * @throws {ApiError} `VALIDATION_ERROR` when the path is missing, unreadable, + * or not a regular file. + */ +export function readSecretFileGuarded(flag: string, path: string): string { + const absolute = isAbsolute(path) ? path : resolve(process.cwd(), path); + + let stat; + try { + stat = statSync(absolute); + } catch (err) { + throw secretFileError(flag, path, err, 'stat'); + } + + // A directory would otherwise reach readFileSync and throw EISDIR on Linux + // while resolving to an empty read on some platforms — reject it up front so + // the contract is the same everywhere. + if (!stat.isFile()) { + throw localValidationError(flag, `not a regular file: ${path}`); + } + + try { + return readFileSync(absolute, 'utf8').trim(); + } catch (err) { + throw secretFileError(flag, path, err, 'read'); + } +} + +/** + * Translate a Node filesystem error into the CLI's validation envelope, + * reporting the path the user typed rather than the resolved absolute path so + * no directory layout leaks into output. + */ +function secretFileError( + flag: string, + path: string, + err: unknown, + verb: 'stat' | 'read', +): ReturnType { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ENOENT') { + return localValidationError(flag, `file does not exist: ${path}`); + } + if (code === 'EACCES' || code === 'EPERM') { + return localValidationError(flag, `permission denied reading ${path}`); + } + if (code === 'EISDIR') { + return localValidationError(flag, `not a regular file: ${path}`); + } + const reason = err instanceof Error ? err.message : 'unknown error'; + return localValidationError(flag, `cannot ${verb} ${path}: ${reason}`); +} From 68ffef6465552f5bcdf46daeaac565d161115c09 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 13 Aug 2026 03:54:54 -0500 Subject: [PATCH 114/117] fix(dry-run): include failed run-scoped step sample (#218) * fix(dry-run): include failed run-scoped step sample * fix(dry-run): use sentinel failed run sample * fix(dry-run): address failed run sample review * fix(dry-run): keep sample body resolution lazy * test(dry-run): preserve cancel sample after rebase --- src/commands/test.test.ts | 34 +++++++ src/lib/dry-run/samples.test.ts | 60 +++++++++-- src/lib/dry-run/samples.ts | 175 ++++++++++++++++++++++---------- 3 files changed, 208 insertions(+), 61 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index d76ee5d..098c461 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -3096,6 +3096,40 @@ describe('runSteps', () => { expect(block.match(/error: /g)).toHaveLength(1); }); + it('--run-id run_failed_sample dry-run sample maps the failed step error and contributor flag', async () => { + const out: string[] = []; + const page = await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + testId: 'test_fe', + runId: 'run_failed_sample', + }, + { + env: {} as NodeJS.ProcessEnv, + credentialsPath: join(tmpdir(), 'testsprite-no-creds'), + stdout: line => out.push(line), + stderr: () => undefined, + }, + ); + + const failing = page.items.find(step => step.stepIndex === 3); + expect(failing).toMatchObject({ + status: 'failed', + error: expect.any(String), + stepType: 'assertion', + outcomeContributesToFailure: true, + }); + expect(page.items.find(step => step.stepIndex === 1)?.outcomeContributesToFailure).toBe(false); + + const printed = JSON.parse(out[0]!) as { items: Array<{ stepIndex: number; error?: string }> }; + expect(printed.items.some(step => step.stepIndex === 3 && typeof step.error === 'string')).toBe( + true, + ); + }); + it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => { const { credentialsPath } = makeCreds(); // The run-scoped endpoint returns a run whose testId differs from the diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index 9f930da..151c6a6 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -445,6 +445,44 @@ describe('findSample', () => { expect(body.stepSummary.failedCount).toBe(0); }); + it('GET /runs/run_failed_sample resolves to the sentinel failed run-scoped step sample', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_failed_sample'); + expect(e?.operationId).toBe('getRun'); + const body = e?.body() as { + status: string; + runId: string; + failedStepIndex: number | null; + failureKind: string | null; + error: string | null; + stepSummary: { total: number; completed: number; passedCount: number; failedCount: number }; + steps: Array<{ + stepIndex: string; + type: string; + status: string | null; + error: string | null; + }>; + }; + expect(body.runId).toBe('run_failed_sample'); + expect(body.status).toBe('failed'); + expect(body.failedStepIndex).toBe(3); + expect(body.failureKind).toBe('assertion'); + expect(body.error).toEqual(expect.any(String)); + expect(body.stepSummary).toMatchObject({ + total: 3, + completed: 3, + passedCount: 2, + failedCount: 1, + }); + + const failingStep = body.steps.find(step => step.stepIndex === '0003'); + expect(failingStep).toMatchObject({ + type: 'assertion', + status: 'failed', + error: expect.any(String), + }); + expect(failingStep?.error).not.toBe(''); + }); + // DEV-331 piece 3: POST /runs/{runId}/cancel must resolve to `cancelRun`, // never fall through to the GET-only `getRun` entry despite sharing the // `/runs/{runId}` path prefix — findSample filters by method first. @@ -562,13 +600,23 @@ describe('findSample', () => { expect(body.summary.total).toBeGreaterThanOrEqual(1); }); - it('only one getRun entry exists in the registry (no duplicate)', () => { - // Guards against re-introducing the duplicate by ensuring exactly one - // sample is registered for GET /runs/{runId}. - const matches = DRY_RUN_SAMPLE_ENTRIES.filter( - e => e.method === 'GET' && e.operationId === 'getRun', + it('keeps the failed run sentinel before generic getRun while retaining cancelRun', () => { + // findSample is first-match-wins; exact run fixtures must precede + // `/runs/{runId}` so `test wait --dry-run` still gets the passed sample. + // The adjacent POST cancel fixture was added on main after this branch forked. + const failedRunIndex = DRY_RUN_SAMPLE_ENTRIES.findIndex( + e => e.method === 'GET' && e.pathTemplate === '/runs/run_failed_sample', + ); + const genericRunIndex = DRY_RUN_SAMPLE_ENTRIES.findIndex( + e => e.method === 'GET' && e.pathTemplate === '/runs/{runId}', + ); + const cancelRunIndex = DRY_RUN_SAMPLE_ENTRIES.findIndex( + e => e.method === 'POST' && e.pathTemplate === '/runs/{runId}/cancel', ); - expect(matches).toHaveLength(1); + expect(failedRunIndex).toBeGreaterThanOrEqual(0); + expect(genericRunIndex).toBeGreaterThanOrEqual(0); + expect(cancelRunIndex).toBeGreaterThanOrEqual(0); + expect(failedRunIndex).toBeLessThan(genericRunIndex); }); // Input-derived sample tests (Fix #1 — dogfood 2026-05-15) diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 84c8e4e..29ffa66 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -53,6 +53,10 @@ const SAMPLE_TEST_ID_FAILED = 'test_8f2a4d10'; const SAMPLE_TEST_ID_PASSED = 'test_3a91bb02'; const SAMPLE_TEST_ID_BLOCKED = 'test_blocked_4f7a'; export const SAMPLE_RUN_ID = 'run_abc'; +// Documented sentinel for `test steps --run-id run_failed_sample --dry-run`: +// keeps wait flows on the default passed sample while still demonstrating a +// run-scoped failed step offline. +const SAMPLE_FAILED_RUN_ID = 'run_failed_sample'; // M3.4 rerun dry-run sample IDs const SAMPLE_RERUN_ID_BE_NAMED = 'run_rerun_be_named'; const SAMPLE_RERUN_ID_BE_PRODUCER = 'run_rerun_be_producer'; @@ -357,6 +361,118 @@ const failureSummary: CliFailureSummary = { recommendedFixTarget: failureContext.failure.recommendedFixTarget, }; +const passedRunSample: RunResponse = { + runId: SAMPLE_RUN_ID, + testId: SAMPLE_TEST_ID_PASSED, + projectId: SAMPLE_PROJECT_ID, + userId: SAMPLE_USER_ID, + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T19:32:00.000Z', + startedAt: '2026-05-15T19:32:05.000Z', + finishedAt: '2026-05-15T19:34:00.000Z', + codeVersion: 'v1', + targetUrl: SAMPLE_TARGET_URL, + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 2, + completed: 2, + passedCount: 2, + failedCount: 0, + }, + // Representative per-run steps so `test steps --run-id --dry-run` + // demonstrates real output instead of an empty list (the generic + // `/runs/{runId}` sample is also used by `test wait`, which ignores steps). + steps: [ + { + stepIndex: '0001', + type: 'action', + action: 'navigate', + status: 'passed', + description: 'Open the target URL', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:10.000Z', + }, + { + stepIndex: '0002', + type: 'assertion', + action: 'assert_visible', + status: 'passed', + description: 'Dashboard heading is visible', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:20.000Z', + }, + ], +}; + +const failedRunSample: RunResponse = { + runId: SAMPLE_FAILED_RUN_ID, + testId: SAMPLE_TEST_ID_FAILED, + projectId: SAMPLE_PROJECT_ID, + userId: SAMPLE_USER_ID, + status: 'failed', + source: 'cli', + createdAt: '2026-05-15T19:32:00.000Z', + startedAt: '2026-05-15T19:32:05.000Z', + finishedAt: '2026-05-15T19:34:00.000Z', + codeVersion: 'v1', + targetUrl: SAMPLE_TARGET_URL, + createdFrom: null, + failedStepIndex: 3, + failureKind: 'assertion', + error: 'Expected billing status badge to be visible, but it was not found.', + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: 2, + failedCount: 1, + }, + steps: [ + { + stepIndex: '0001', + type: 'action', + action: 'navigate', + status: 'passed', + description: 'Open the target URL', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:10.000Z', + }, + { + stepIndex: '0002', + type: 'assertion', + action: 'assert_visible', + status: 'passed', + description: 'Dashboard heading is visible', + error: null, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:20.000Z', + }, + { + stepIndex: '0003', + type: 'assertion', + action: 'assert_visible', + status: 'failed', + description: 'Billing status badge is visible', + error: 'Expected billing status badge to be visible, but it was not found.', + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-05-15T19:32:30.000Z', + }, + ], +}; + /** * Dry-run sample lookup keyed by OpenAPI operationId. Order matters in * {@link findSample}: more specific patterns must precede their generic @@ -722,57 +838,8 @@ const ENTRIES: DryRunSampleEntry[] = [ // fix(2026-05-21): a duplicate failed-shape entry that appeared before // this entry was removed; findSample first-match-wins was always // returning status: "failed" for `test wait --dry-run`. - entry('getRun', 'GET', '/runs/{runId}', { - runId: SAMPLE_RUN_ID, - testId: SAMPLE_TEST_ID_PASSED, - projectId: SAMPLE_PROJECT_ID, - userId: SAMPLE_USER_ID, - status: 'passed', - source: 'cli', - createdAt: '2026-05-15T19:32:00.000Z', - startedAt: '2026-05-15T19:32:05.000Z', - finishedAt: '2026-05-15T19:34:00.000Z', - codeVersion: 'v1', - targetUrl: SAMPLE_TARGET_URL, - createdFrom: null, - failedStepIndex: null, - failureKind: null, - error: null, - videoUrl: null, - stepSummary: { - total: 8, - completed: 8, - passedCount: 8, - failedCount: 0, - }, - // Representative per-run steps so `test steps --run-id --dry-run` - // demonstrates real output instead of an empty list (the generic - // `/runs/{runId}` sample is also used by `test wait`, which ignores steps). - steps: [ - { - stepIndex: '0001', - type: 'action', - action: 'navigate', - status: 'passed', - description: 'Open the target URL', - error: null, - screenshotUrl: null, - htmlSnapshotUrl: null, - createdAt: '2026-05-15T19:32:10.000Z', - }, - { - stepIndex: '0002', - type: 'assertion', - action: 'assert_visible', - status: 'passed', - description: 'Dashboard heading is visible', - error: null, - screenshotUrl: null, - htmlSnapshotUrl: null, - createdAt: '2026-05-15T19:32:20.000Z', - }, - ], - } satisfies RunResponse), + entry('getRun', 'GET', `/runs/${SAMPLE_FAILED_RUN_ID}`, failedRunSample), + entry('getRun', 'GET', '/runs/{runId}', passedRunSample), // DEV-331 piece 3 — POST /runs/{runId}/cancel. Method-guarded in // `findSample` (POST vs `getRun`'s GET), so this can't be shadowed by the // broader `/runs/{runId}` pattern above despite sharing its path prefix. @@ -854,10 +921,8 @@ export function findSample( const pathOnly = extractPath(url); for (const e of ENTRIES) { if (e.method === upper && e.pattern.test(pathOnly)) { - // Rebind body so callers get the resolved value, not the factory. - // We return a new object with `body` already applied so downstream - // code can keep calling `e.body` as-before (no API break for tests - // that call `findSample` directly). + // Rebind body so downstream code can call `e.body` as before while + // still preserving the original lazy factory semantics. return { ...e, body: () => e.body(requestBody) }; } } From f274451f1ad6202fc3421a68dafc34bf1994ded0 Mon Sep 17 00:00:00 2001 From: nopp Date: Thu, 13 Aug 2026 15:55:03 +0700 Subject: [PATCH 115/117] test: cover cross-process credential writes (#280) * test: cover cross-process credential writes * test: cover credential writer failure paths * test: make credential writer race deterministic --- test/credentials-cross-process.test.ts | 181 +++++++++++++++++++++++ test/helpers/credentials-write-child.mjs | 45 ++++++ 2 files changed, 226 insertions(+) create mode 100644 test/credentials-cross-process.test.ts create mode 100644 test/helpers/credentials-write-child.mjs diff --git a/test/credentials-cross-process.test.ts b/test/credentials-cross-process.test.ts new file mode 100644 index 0000000..3e00d14 --- /dev/null +++ b/test/credentials-cross-process.test.ts @@ -0,0 +1,181 @@ +import { spawn } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { readCredentialsFile } from '../src/lib/credentials.js'; +import { execNpm } from './helpers/execNpm.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, '..'); +const CHILD_PATH = join(REPO_ROOT, 'test', 'helpers', 'credentials-write-child.mjs'); + +interface ChildResult { + code: number | null; + signal: NodeJS.Signals | null; + stderr: string; + stdout: string; +} + +function runCredentialWriter(env: Record): Promise { + return new Promise((resolveChild, reject) => { + const child = spawn(process.execPath, [CHILD_PATH], { + cwd: REPO_ROOT, + env: { ...process.env, ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + const timer = setTimeout(() => { + child.kill(); + }, 10_000); + + child.stdout.on('data', chunk => stdout.push(Buffer.from(chunk))); + child.stderr.on('data', chunk => stderr.push(Buffer.from(chunk))); + child.on('error', error => { + clearTimeout(timer); + reject(error); + }); + child.on('close', (code, signal) => { + clearTimeout(timer); + resolveChild({ + code, + signal, + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8'), + }); + }); + }); +} + +async function waitForFiles(paths: string[], timeoutMs = 5_000): Promise { + const deadline = Date.now() + timeoutMs; + + while (true) { + const missing = paths.filter(path => !existsSync(path)); + if (missing.length === 0) return; + if (Date.now() >= deadline) { + throw new Error(`Timed out waiting for files: ${missing.join(', ')}`); + } + await new Promise(resolveDelay => setTimeout(resolveDelay, 5)); + } +} + +describe('credentials cross-process writes', () => { + beforeAll(() => { + execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + }, 60_000); + + it('reports missing required child-process environment variables', async () => { + const result = await runCredentialWriter({ + CRED_API_KEY: '', + CRED_PATH: '', + CRED_PROFILE: '', + CRED_START_PATH: '', + }); + + expect(result).toMatchObject({ + code: 1, + signal: null, + stdout: '', + }); + expect(result.stderr).toContain( + 'CRED_PROFILE, CRED_PATH, CRED_API_KEY, and CRED_START_PATH are required', + ); + }); + + it('reports a timed out child-process start marker', async () => { + const tmpRoot = mkdtempSync(join(tmpdir(), 'testsprite-creds-timeout-')); + + try { + const startPath = join(tmpRoot, 'missing-start'); + const result = await runCredentialWriter({ + CRED_API_KEY: 'sk-child-timeout', + CRED_PATH: join(tmpRoot, 'credentials'), + CRED_PROFILE: 'child-timeout', + CRED_POLL_MS: '1', + CRED_START_PATH: startPath, + CRED_START_TIMEOUT_MS: '100', + }); + + expect(result).toMatchObject({ + code: 2, + signal: null, + stdout: '', + }); + expect(result.stderr).toContain(`Timed out waiting for start marker: ${startPath}`); + } finally { + rmSync(tmpRoot, { force: true, recursive: true }); + } + }, 15_000); + + it('reports writeProfile failures from the child process', async () => { + const tmpRoot = mkdtempSync(join(tmpdir(), 'testsprite-creds-write-error-')); + + try { + const startPath = join(tmpRoot, 'start'); + writeFileSync(startPath, 'go'); + + const result = await runCredentialWriter({ + CRED_API_KEY: 'sk-child-invalid-profile', + CRED_PATH: join(tmpRoot, 'credentials'), + CRED_PROFILE: 'bad]', + CRED_START_PATH: startPath, + }); + + expect(result).toMatchObject({ + code: 3, + signal: null, + stdout: '', + }); + expect(result.stderr).toContain('Invalid request.'); + } finally { + rmSync(tmpRoot, { force: true, recursive: true }); + } + }); + + it('preserves every profile written by concurrent child processes', async () => { + const tmpRoot = mkdtempSync(join(tmpdir(), 'testsprite-creds-race-')); + const credentialsPath = join(tmpRoot, 'credentials'); + const startPath = join(tmpRoot, 'start'); + + try { + const profiles = Array.from({ length: 8 }, (_, index) => ({ + apiKey: `sk-child-${index}`, + profile: `child-${index}`, + })); + const readyPaths = profiles.map(({ profile }) => join(tmpRoot, `${profile}.ready`)); + const children = profiles.map(({ apiKey, profile }, index) => + runCredentialWriter({ + CRED_API_KEY: apiKey, + CRED_PATH: credentialsPath, + CRED_PROFILE: profile, + CRED_READY_PATH: readyPaths[index]!, + CRED_START_PATH: startPath, + }), + ); + + await waitForFiles(readyPaths); + writeFileSync(startPath, 'go'); + + const results = await Promise.all(children); + expect(results).toEqual( + profiles.map(() => ({ + code: 0, + signal: null, + stderr: '', + stdout: '', + })), + ); + + const credentials = readCredentialsFile({ path: credentialsPath }); + for (const { apiKey, profile } of profiles) { + expect(credentials[profile]).toEqual({ apiKey }); + } + expect(existsSync(`${credentialsPath}.lock`)).toBe(false); + } finally { + rmSync(tmpRoot, { force: true, recursive: true }); + } + }, 20_000); +}); diff --git a/test/helpers/credentials-write-child.mjs b/test/helpers/credentials-write-child.mjs new file mode 100644 index 0000000..2b6072a --- /dev/null +++ b/test/helpers/credentials-write-child.mjs @@ -0,0 +1,45 @@ +import { existsSync, writeFileSync } from 'node:fs'; +import { writeProfile } from '../../dist/lib/credentials.js'; + +const profile = process.env.CRED_PROFILE; +const credentialsPath = process.env.CRED_PATH; +const apiKey = process.env.CRED_API_KEY; +const startPath = process.env.CRED_START_PATH; +const readyPath = process.env.CRED_READY_PATH; +const startTimeoutMs = Number(process.env.CRED_START_TIMEOUT_MS ?? '5000'); +const pollMs = Number(process.env.CRED_POLL_MS ?? '5'); + +if (!profile || !credentialsPath || !apiKey || !startPath) { + console.error('CRED_PROFILE, CRED_PATH, CRED_API_KEY, and CRED_START_PATH are required'); + process.exit(1); +} + +if ( + !Number.isInteger(startTimeoutMs) || + startTimeoutMs < 1 || + !Number.isInteger(pollMs) || + pollMs < 1 +) { + console.error('CRED_START_TIMEOUT_MS and CRED_POLL_MS must be positive integers'); + process.exit(1); +} + +if (readyPath) { + writeFileSync(readyPath, 'ready'); +} + +const deadline = Date.now() + startTimeoutMs; +while (!existsSync(startPath)) { + if (Date.now() >= deadline) { + console.error(`Timed out waiting for start marker: ${startPath}`); + process.exit(2); + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, pollMs); +} + +try { + writeProfile(profile, { apiKey }, { path: credentialsPath }); +} catch (error) { + console.error(error instanceof Error ? error.stack : String(error)); + process.exit(3); +} From 97bcd7a0334becfdede8a5c4e35d26294b892eef Mon Sep 17 00:00:00 2001 From: zengxm1979 Date: Thu, 13 Aug 2026 16:55:11 +0800 Subject: [PATCH 116/117] fix(test): align mock latest result schema (#290) --- test/contract/p4-schema.test.ts | 26 ++++++++++++++------------ test/mock-backend/fixtures.ts | 16 ++++++++++++---- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/test/contract/p4-schema.test.ts b/test/contract/p4-schema.test.ts index c90e5f9..cedea9c 100644 --- a/test/contract/p4-schema.test.ts +++ b/test/contract/p4-schema.test.ts @@ -83,6 +83,8 @@ const LATEST_RESULT_REQUIRED = [ 'targetUrl', 'failedStepIndex', 'failureKind', + 'verdict', + 'executionStatus', 'summary', ] as const; // `targetUrlSource` (D1) is OPTIONAL — present on backends that shipped the D1 @@ -109,6 +111,15 @@ const FAILURE_KINDS = new Set([ 'unknown', null, ]); +const VERDICTS = new Set(['passed', 'failed', 'blocked', 'cancelled', 'unknown', null]); +const EXECUTION_STATUSES = new Set([ + 'queued', + 'running', + 'completed', + 'cancelled', + 'error', + 'unknown', +]); function expectKeysMatch(value: Record, allowed: Set, label: string) { for (const key of Object.keys(value)) { @@ -178,17 +189,6 @@ function validateTestStepList(value: unknown, label = 'TestStepList'): void { } } -function validateResultSummary(value: unknown, label: string): void { - expect(value, label).toBeTypeOf('object'); - expect(value, label).not.toBeNull(); - const obj = value as Record; - expectKeysMatch(obj, new Set(['passed', 'failed', 'skipped']), label); - for (const k of ['passed', 'failed', 'skipped']) { - expect(typeof obj[k], `${label}.${k}`).toBe('number'); - expect((obj[k] as number) >= 0, `${label}.${k} >= 0`).toBe(true); - } -} - function validateLatestResult(value: unknown, label = 'LatestResult'): void { expect(value, `${label}: must be an object`).toBeTypeOf('object'); expect(value, `${label}: must not be null`).not.toBeNull(); @@ -215,7 +215,9 @@ function validateLatestResult(value: unknown, label = 'LatestResult'): void { expect((obj.failedStepIndex as number) >= 1, `${label}.failedStepIndex >= 1`).toBe(true); } expect(FAILURE_KINDS.has(obj.failureKind), `${label}.failureKind`).toBe(true); - validateResultSummary(obj.summary, `${label}.summary`); + expect(VERDICTS.has(obj.verdict), `${label}.verdict`).toBe(true); + expect(EXECUTION_STATUSES.has(obj.executionStatus), `${label}.executionStatus`).toBe(true); + expect(typeof obj.summary, `${label}.summary`).toBe('string'); } describe('P4 schema contract — fixtures match the OpenAPI shapes', () => { diff --git a/test/mock-backend/fixtures.ts b/test/mock-backend/fixtures.ts index 0722549..0457c54 100644 --- a/test/mock-backend/fixtures.ts +++ b/test/mock-backend/fixtures.ts @@ -243,7 +243,9 @@ export const latestResultRunningFixture = { targetUrl: FIXTURE_TARGET_URL, failedStepIndex: null, failureKind: null, - summary: { passed: 0, failed: 0, skipped: 0 }, + verdict: null, + executionStatus: 'running' as const, + summary: 'Run is still in progress.', }; export const latestResultPassedFixture = { @@ -260,7 +262,9 @@ export const latestResultPassedFixture = { targetUrlSource: 'run' as const, failedStepIndex: null, failureKind: null, - summary: { passed: 8, failed: 0, skipped: 0 }, + verdict: 'passed' as const, + executionStatus: 'completed' as const, + summary: 'Passed all 8 steps.', }; export const latestResultFailedFixture = { @@ -277,7 +281,9 @@ export const latestResultFailedFixture = { targetUrlSource: 'run' as const, failedStepIndex: 5, failureKind: 'assertion' as const, - summary: { passed: 4, failed: 1, skipped: 0 }, + verdict: 'failed' as const, + executionStatus: 'completed' as const, + summary: 'Failed (assertion) on step 5: expected cart badge to show 1 item, but it was empty.', }; export const failureContextFixture = { @@ -351,7 +357,9 @@ export const failureContextNoAnalysisFixture = { targetUrl: 'https://staging.example.com/', failedStepIndex: 2, failureKind: 'unknown' as const, - summary: { passed: 1, failed: 1, skipped: 0 }, + verdict: 'failed' as const, + executionStatus: 'completed' as const, + summary: 'Failed with no detailed analysis available.', }, steps: [], code: { From 428223ec800c371811edfca837db096d0c0e1af2 Mon Sep 17 00:00:00 2001 From: Yazan-O <154028854+Yazan-O@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:27:15 -0500 Subject: [PATCH 117/117] fix(http): bound the size of buffered JSON responses (#281) response.json() buffers the entire body before parsing with no upper bound, so a large test result --history page or a run with a long steps[] array grows the heap in proportion to the payload. Read the typed JSON response bounded by a configurable cap (HttpClientOptions.maxResponseBytes, default 64 MiB): reject an over-cap Content-Length up front, count bytes while streaming chunked bodies, and throw a typed PAYLOAD_TOO_LARGE (exit 5) with guidance to narrow --page-size / --since. --- src/lib/http.test.ts | 60 +++++++++++++++++++- src/lib/http.ts | 127 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 185 insertions(+), 2 deletions(-) diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 0626ee1..a857c0a 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -33,6 +33,7 @@ function makeClient( apiKey?: string | null; onDebug?: (e: DebugEvent) => void; onServerVersion?: (info: { minVersion?: string }) => void; + maxResponseBytes?: number; } = {}, ): HttpClient { const apiKey = 'apiKey' in options ? (options.apiKey ?? undefined) : 'sk-test'; @@ -44,9 +45,61 @@ function makeClient( random: () => 0, onDebug: options.onDebug, onServerVersion: options.onServerVersion, + maxResponseBytes: options.maxResponseBytes, }); } +describe('response size guard (maxResponseBytes)', () => { + it('rejects a response whose Content-Length exceeds the cap', async () => { + // jsonResponse sets a real Content-Length; a 50-byte cap is well under it. + const fetchImpl = vi.fn().mockResolvedValue(jsonResponse({ blob: 'x'.repeat(500) })); + const client = makeClient(fetchImpl as unknown as typeof fetch, { maxResponseBytes: 50 }); + + const err = await client.get('/tests').catch((e: unknown) => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('PAYLOAD_TOO_LARGE'); + expect((err as ApiError).exitCode).toBe(5); + expect((err as ApiError).getDetail('maxBytes')).toBe(50); + }); + + it('rejects an over-cap chunked response that has no Content-Length', async () => { + // A body built from a ReadableStream carries no Content-Length, so only the + // streaming byte-counter can catch it. + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('{"data":"')); + controller.enqueue(encoder.encode('y'.repeat(500))); + controller.enqueue(encoder.encode('"}')); + controller.close(); + }, + }); + const fetchImpl = vi + .fn() + .mockResolvedValue( + new Response(stream, { status: 200, headers: { 'content-type': 'application/json' } }), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch, { maxResponseBytes: 50 }); + + const err = await client.get('/tests').catch((e: unknown) => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('PAYLOAD_TOO_LARGE'); + }); + + it('reads a within-cap response unchanged', async () => { + const fetchImpl = vi.fn().mockResolvedValue(jsonResponse({ ok: true, items: [1, 2, 3] })); + const client = makeClient(fetchImpl as unknown as typeof fetch, { + maxResponseBytes: 1_000_000, + }); + + const body = await client.get<{ ok: boolean; items: number[] }>('/tests'); + + expect(body).toEqual({ ok: true, items: [1, 2, 3] }); + }); +}); + describe('CLIENT_TOO_OLD (426)', () => { it('is not retried — fails fast with the typed error', async () => { const fetchImpl = vi.fn().mockResolvedValue(errorEnvelopeResponse(426, 'CLIENT_TOO_OLD')); @@ -745,7 +798,12 @@ describe('HttpClient per-request timeout', () => { return { ok: true, status: 200, - json: () => Promise.reject(timeoutErr), + headers: new Headers(), + body: new ReadableStream({ + start(controller) { + controller.error(timeoutErr); + }, + }), } as unknown as Response; }); const client = new HttpClient({ diff --git a/src/lib/http.ts b/src/lib/http.ts index 1af1f51..64a50e0 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -115,6 +115,14 @@ export interface HttpClientOptions { * `globalShutdown.signal` via the client factory. */ shutdownSignal?: AbortSignal; + /** + * Upper bound (bytes) on a successful JSON response body the client will + * buffer before parsing. A response whose declared `Content-Length` — or + * actual streamed size — exceeds this fails fast with a typed + * `PAYLOAD_TOO_LARGE` error instead of growing the heap without limit. + * Defaults to {@link MAX_RESPONSE_BYTES_DEFAULT} (64 MiB). + */ + maxResponseBytes?: number; } export interface RequestOptions { @@ -193,6 +201,15 @@ const MAX_RATE_LIMITED_DELAY_MS = 60_000; const CONFLICT_DELAY_MS = 1000; const INTERNAL_DELAY_MS = 500; +// Upper bound on the size of a successful JSON response body the client will +// buffer into memory. `response.json()` reads the ENTIRE body before parsing, +// with no limit, so a large `test result --history` page or a run with a long +// `steps[]` array (getRun `includeSteps`) would grow the heap in proportion to +// the payload. 64 MiB sits far above any legitimate metadata/history/steps +// response yet still bounds a pathological — or hostile — one. Override per +// client via `HttpClientOptions.maxResponseBytes`. +const MAX_RESPONSE_BYTES_DEFAULT = 64 * 1024 * 1024; + // Cap on how many valibot issues a shape-mismatch INTERNAL envelope carries in // `details.issues` (path + message each). Keeps the envelope readable and // guarantees the response body itself is never echoed back to the operator. @@ -219,6 +236,7 @@ export class HttpClient { private readonly onServerVersion?: (info: { minVersion?: string }) => void; private readonly requestTimeoutMs: number; private readonly shutdownSignal?: AbortSignal; + private readonly maxResponseBytes: number; constructor(options: HttpClientOptions) { this.baseUrl = trimTrailingSlash(options.baseUrl); @@ -231,6 +249,7 @@ export class HttpClient { this.onTransition = options.onTransition; this.onServerVersion = options.onServerVersion; this.requestTimeoutMs = options.requestTimeoutMs ?? REQUEST_TIMEOUT_DEFAULT_MS; + this.maxResponseBytes = options.maxResponseBytes ?? MAX_RESPONSE_BYTES_DEFAULT; } /** @@ -649,10 +668,17 @@ export class HttpClient { }); let raw: unknown; try { - raw = await response.json(); + // Bounded read, then parse — assigning to `raw` rather than returning + // here keeps the schema validation below on the success path. + const text = await readBoundedText(response, this.maxResponseBytes, requestId); + raw = JSON.parse(text); } catch (err) { // Interrupt passthrough (see the fetch catch above). if (err instanceof InterruptError) throw err; + // A bounded-read rejection (PAYLOAD_TOO_LARGE) — or any typed ApiError — + // is a real, actionable outcome; surface it unchanged rather than + // masking it as a malformed-body error below. + if (err instanceof ApiError) throw err; // A timeout/abort can fire mid-body-read (headers received, stream stalls). this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); // Otherwise the successful response body was not valid JSON — a @@ -1068,6 +1094,105 @@ export function malformedResponseError( ); } +/** + * Read a successful response body as text, bounded to `maxBytes`. + * + * `response.json()` buffers the ENTIRE body into memory before parsing, with no + * upper limit — a large `test result --history` page, or a run with a long + * `steps[]` array (getRun `includeSteps`), grows the heap in proportion to the + * payload. This reads the body incrementally and stops once the accumulated + * size crosses `maxBytes`, so a pathological (or hostile) response fails fast + * with a typed PAYLOAD_TOO_LARGE error instead of exhausting memory. + * + * A declared `Content-Length` over the cap is rejected before any body is read. + * Chunked bodies (no `Content-Length`) are bounded by counting bytes as they + * stream. Decoding happens once, at the end, so multi-byte UTF-8 sequences that + * straddle a chunk boundary still decode correctly. + */ +async function readBoundedText( + response: Response, + maxBytes: number, + requestId: string, +): Promise { + const declared = Number(response.headers.get('content-length')); + if (Number.isFinite(declared) && declared > maxBytes) { + throw responseTooLargeError(requestId, maxBytes, declared); + } + const stream = response.body; + if (stream === null) { + // No readable stream exposed (some runtimes / test doubles): fall back to a + // buffered read, then enforce the cap on the materialized text. + const text = await response.text(); + if (new TextEncoder().encode(text).length > maxBytes) { + throw responseTooLargeError(requestId, maxBytes); + } + return text; + } + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + if (value === undefined) continue; + total += value.byteLength; + if (total > maxBytes) { + await reader.cancel(); + throw responseTooLargeError(requestId, maxBytes, total); + } + chunks.push(value); + } + } finally { + try { + reader.releaseLock(); + } catch { + // The reader may already be released after cancel(); releasing twice is a + // no-op we don't want surfacing over the original error. + } + } + return new TextDecoder('utf-8').decode(concatChunks(chunks, total)); +} + +/** Concatenate byte chunks into a single `Uint8Array` of known total length. */ +function concatChunks(chunks: readonly Uint8Array[], total: number): Uint8Array { + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.byteLength; + } + return out; +} + +/** + * Typed error for a response body that exceeds the client-side buffering cap. + * Reuses PAYLOAD_TOO_LARGE (exit 5, validation family) — the same code the + * backend returns for oversized request bodies — so machine consumers route on + * it uniformly. `nextAction` names the knobs that shrink the result set. + */ +function responseTooLargeError( + requestId: string, + maxBytes: number, + observedBytes?: number, +): ApiError { + const limitMiB = Math.round(maxBytes / (1024 * 1024)); + return new ApiError({ + code: 'PAYLOAD_TOO_LARGE', + message: + `The server response exceeded the client-side ${limitMiB} MiB limit and was not ` + + `buffered, to avoid unbounded memory use.`, + nextAction: + 'Narrow the result set and retry — e.g. a smaller --page-size, a tighter --since ' + + 'window, or scope --history to a single run.', + requestId, + details: { + maxBytes, + ...(observedBytes !== undefined ? { observedBytes } : {}), + }, + }); +} + export function parseRetryAfter(headerValue: string | null): number | undefined { if (!headerValue) return undefined; const numeric = Number(headerValue);