From e44854c281a4ad799b353328caa0952833553106 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Fri, 10 Apr 2026 13:22:56 -0400 Subject: [PATCH 1/2] Surface skipped/todo QUnit tests in TestRun results instead of hiding them as passed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QUnit skipped and todo tests were previously mapped to 'passed', creating a blind spot where the agent could write QUnit.skip() tests and see "all passed". Now they are mapped to a new 'skipped' status that is: - Terminal (not pending) so resume/isComplete logic still works - Visually distinct in the TestRun card UI (→ icon, muted italic styling) - Surfaced in counts (skippedCount on TestModuleResult, TestRun, TestResult) - Treated as failure when ALL tests are skipped (nothing actually verified) - Included in formatTestResultSummary and validation context for the agent - Documented in SKILL.md to tell the agent not to use QUnit.skip()/todo() Closes CS-10651 Co-Authored-By: Claude Opus 4.6 (1M context) --- .../software-factory-operations/SKILL.md | 1 + .../software-factory/realm/test-results.gts | 91 +++++++++++++++- .../src/factory-agent-types.ts | 1 + .../src/factory-prompt-loader.ts | 1 + .../software-factory/src/test-run-parsing.ts | 36 ++++-- .../software-factory/src/test-run-types.ts | 3 +- .../src/validators/test-step.ts | 14 ++- .../tests/factory-test-realm.test.ts | 103 ++++++++++++++++++ 8 files changed, 236 insertions(+), 14 deletions(-) diff --git a/packages/software-factory/.agents/skills/software-factory-operations/SKILL.md b/packages/software-factory/.agents/skills/software-factory-operations/SKILL.md index 7bcd6583832..ce78ba4581e 100644 --- a/packages/software-factory/.agents/skills/software-factory-operations/SKILL.md +++ b/packages/software-factory/.agents/skills/software-factory-operations/SKILL.md @@ -134,6 +134,7 @@ export function runTests() { - No external realm writes during tests — all test data lives in browser memory - Use `data-test-*` attributes for DOM selectors when testing rendered output - Use QUnit assertions: `assert.dom()`, `assert.strictEqual()`, `assert.ok()` +- **Never use `QUnit.skip()` or `QUnit.todo()`.** All tests must actually execute. Skipped/todo tests are flagged as `skipped` in the TestRun card and treated as a failure when no tests actually ran. The orchestrator will reject a TestRun where every test is skipped. ## Important Rules diff --git a/packages/software-factory/realm/test-results.gts b/packages/software-factory/realm/test-results.gts index 9452126507f..98283022cc6 100644 --- a/packages/software-factory/realm/test-results.gts +++ b/packages/software-factory/realm/test-results.gts @@ -29,6 +29,7 @@ export const TestResultStatusField = enumField(StringField, { { value: 'passed', label: 'Passed' }, { value: 'failed', label: 'Failed' }, { value: 'error', label: 'Error' }, + { value: 'skipped', label: 'Skipped' }, ], }); @@ -50,6 +51,8 @@ export class TestResultEntry extends FieldDef { return '\u2717'; case 'error': return '!'; + case 'skipped': + return '\u2192'; case 'pending': return '\u2013'; default: @@ -90,6 +93,10 @@ export class TestResultEntry extends FieldDef { .status-pending { color: var(--boxel-400, #9ca3af); } + .status-skipped { + color: var(--boxel-400, #9ca3af); + font-style: italic; + } .test-name { flex: 1; } @@ -122,6 +129,12 @@ export class TestModuleResult extends FieldDef { }, }); + @field skippedCount = contains(NumberField, { + computeVia: function (this: TestModuleResult) { + return (this.results ?? []).filter((r) => r.status === 'skipped').length; + }, + }); + get moduleName() { return this.moduleRef?.module ?? 'default'; } @@ -145,6 +158,12 @@ export class TestModuleResult extends FieldDef { {{this.args.model.passedCount}}/{{this.total}} passed + {{#if this.args.model.skippedCount}} + + ({{this.args.model.skippedCount}} + skipped) + + {{/if}} {{else}} running... @@ -174,6 +193,10 @@ export class TestModuleResult extends FieldDef { font-size: 0.8rem; color: var(--muted-foreground); } + .skipped-label { + color: var(--boxel-400, #9ca3af); + font-style: italic; + } .module-entries { padding-left: 0.5rem; } @@ -213,6 +236,15 @@ export class TestRun extends CardDef { }, }); + @field skippedCount = contains(NumberField, { + computeVia: function (this: TestRun) { + return (this.moduleResults ?? []).reduce( + (sum, sr) => sum + (sr.skippedCount ?? 0), + 0, + ); + }, + }); + @field title = contains(StringField, { computeVia: function (this: TestRun) { let seq = this.sequenceNumber ?? '?'; @@ -224,7 +256,9 @@ export class TestRun extends CardDef { static fitted = class Fitted extends Component { get total() { return ( - (this.args.model.passedCount ?? 0) + (this.args.model.failedCount ?? 0) + (this.args.model.passedCount ?? 0) + + (this.args.model.failedCount ?? 0) + + (this.args.model.skippedCount ?? 0) ); } @@ -237,6 +271,12 @@ export class TestRun extends CardDef {
{{@model.passedCount}}/{{this.total}} passed + {{#if @model.skippedCount}} + + ({{@model.skippedCount}} + skipped) + + {{/if}} {{#if @model.durationMs}} {{@model.durationMs}}ms {{/if}} @@ -285,6 +325,10 @@ export class TestRun extends CardDef { font-size: 0.85rem; color: var(--muted-foreground); } + .skipped-label { + color: var(--boxel-400, #9ca3af); + font-style: italic; + } .duration { margin-left: 0.5rem; font-size: 0.75rem; @@ -298,7 +342,9 @@ export class TestRun extends CardDef { static isolated = class Isolated extends Component { get total() { return ( - (this.args.model.passedCount ?? 0) + (this.args.model.failedCount ?? 0) + (this.args.model.passedCount ?? 0) + + (this.args.model.failedCount ?? 0) + + (this.args.model.skippedCount ?? 0) ); } @@ -310,6 +356,12 @@ export class TestRun extends CardDef { ); } + get skippedResults() { + return (this.args.model.moduleResults ?? []).flatMap((sr) => + (sr.results ?? []).filter((r) => r.status === 'skipped'), + ); + } +