From 9e847138597ad7f85e46e31c443136bc45ed01d0 Mon Sep 17 00:00:00 2001 From: roxblnfk Date: Fri, 3 Jul 2026 16:49:51 +0400 Subject: [PATCH 1/2] fix(runner): fail the run when a test is aborted by a pipeline error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An interceptor that throws (before or after $next) makes TestRunner mark the test Status::Aborted. The terminal/JUnit/TeamCity reporters already render an abort as a failure, yet the process exit code stayed 0: Aborted is not a Status::isFailure(), so CaseRunner/SuiteRunner never escalated. CaseRunner now treats an aborted test as a case failure, so the suite/run status and the exit code agree with what the reporters print. Kept out of Status::isFailure() on purpose — that predicate also drives retry (RetryPolicyRunInterceptor), which must not retry an aborted test. Also give Aborted its own red "A" glyph in the terminal (verbose + dots) so it reads distinctly from a genuine Error ("E"). Adds a self-test (PipelineFailureStatusTest) plus a small stub apparatus: a FailPipeline attribute that self-binds a throwing interceptor via FallbackInterceptor (no plugin needed), covering test-level and case-level throws before/after $next(). Assisted-By: Claude Opus 4.8 (1M context) --- .../Internal/Runner/CaseRunner.php | 7 +- core/Output/Terminal/Renderer/DotSymbol.php | 1 + core/Output/Terminal/Renderer/Formatter.php | 6 +- core/Output/Terminal/Renderer/Symbol.php | 1 + .../Runner/PipelineFailureStatusTest.php | 147 ++++++++++++++++++ .../Stub/Pipeline/CaseAfterScenario.php | 28 ++++ .../Stub/Pipeline/CaseBeforeScenario.php | 27 ++++ .../Stub/Pipeline/FailPipeline.php | 29 ++++ tests/Application/Stub/Pipeline/FailStage.php | 26 ++++ .../Stub/Pipeline/FailingInterceptor.php | 65 ++++++++ .../Stub/Pipeline/TestStageScenarios.php | 36 +++++ 11 files changed, 370 insertions(+), 3 deletions(-) create mode 100644 tests/Application/Feature/Runner/PipelineFailureStatusTest.php create mode 100644 tests/Application/Stub/Pipeline/CaseAfterScenario.php create mode 100644 tests/Application/Stub/Pipeline/CaseBeforeScenario.php create mode 100644 tests/Application/Stub/Pipeline/FailPipeline.php create mode 100644 tests/Application/Stub/Pipeline/FailStage.php create mode 100644 tests/Application/Stub/Pipeline/FailingInterceptor.php create mode 100644 tests/Application/Stub/Pipeline/TestStageScenarios.php diff --git a/core/Application/Internal/Runner/CaseRunner.php b/core/Application/Internal/Runner/CaseRunner.php index fe3d8e09..c8497247 100644 --- a/core/Application/Internal/Runner/CaseRunner.php +++ b/core/Application/Internal/Runner/CaseRunner.php @@ -75,7 +75,12 @@ public function run(CaseInfo $info): CaseResult ); $result = $this->testRunner->runTest($testInfo); - $result->status->isFailure() and $status = Status::Failed; + # A test aborted by a critical interceptor failure has an unknown verdict — treat it as + # a case failure so it propagates up to the suite/run status and the process exit code + # (every reporter already renders an abort as a failure). Deliberately broader than + # Status::isFailure(), which must keep Aborted out so retry/repeat don't retry an abort. + ($result->status->isFailure() || $result->status === Status::Aborted) + and $status = Status::Failed; $results[] = $result; } catch (\Throwable $throwable) { diff --git a/core/Output/Terminal/Renderer/DotSymbol.php b/core/Output/Terminal/Renderer/DotSymbol.php index c3afff18..57ebf5f4 100644 --- a/core/Output/Terminal/Renderer/DotSymbol.php +++ b/core/Output/Terminal/Renderer/DotSymbol.php @@ -14,4 +14,5 @@ enum DotSymbol: string case Skipped = '-'; case Risky = 'R'; case Error = 'E'; + case Aborted = 'A'; } diff --git a/core/Output/Terminal/Renderer/Formatter.php b/core/Output/Terminal/Renderer/Formatter.php index 12a8389d..5917715a 100644 --- a/core/Output/Terminal/Renderer/Formatter.php +++ b/core/Output/Terminal/Renderer/Formatter.php @@ -404,7 +404,8 @@ private static function formatDotRun(FormattedItem $item): string Status::Passed => DotSymbol::Passed->value, Status::Failed => Style::error(DotSymbol::Failed->value), Status::Skipped => Style::warning(DotSymbol::Skipped->value), - Status::Error, Status::Aborted => Style::error(DotSymbol::Error->value), + Status::Error => Style::error(DotSymbol::Error->value), + Status::Aborted => Style::error(DotSymbol::Aborted->value), Status::Risky => Style::warning(DotSymbol::Risky->value), Status::Flaky => Style::info(DotSymbol::Passed->value), Status::Cancelled => Style::dim(DotSymbol::Skipped->value), @@ -422,7 +423,8 @@ private static function getStatusSymbol(Status $status): string Status::Passed => Style::success(Symbol::Success->value), Status::Failed => Style::error(Symbol::Failure->value), Status::Skipped => Style::warning(Symbol::Skipped->value), - Status::Error, Status::Aborted => Style::error(Symbol::Error->value), + Status::Error => Style::error(Symbol::Error->value), + Status::Aborted => Style::error(Symbol::Aborted->value), Status::Risky => Style::warning(Symbol::Risky->value), Status::Flaky => Style::info(Symbol::Flaky->value), Status::Cancelled => Style::dim(Symbol::Skipped->value), diff --git a/core/Output/Terminal/Renderer/Symbol.php b/core/Output/Terminal/Renderer/Symbol.php index 892bbf8b..92eb4b54 100644 --- a/core/Output/Terminal/Renderer/Symbol.php +++ b/core/Output/Terminal/Renderer/Symbol.php @@ -15,5 +15,6 @@ enum Symbol: string case Risky = '?'; case Flaky = '~'; case Error = 'E'; + case Aborted = 'A'; case DataProvider = '◆'; } diff --git a/tests/Application/Feature/Runner/PipelineFailureStatusTest.php b/tests/Application/Feature/Runner/PipelineFailureStatusTest.php new file mode 100644 index 00000000..7845c1c2 --- /dev/null +++ b/tests/Application/Feature/Runner/PipelineFailureStatusTest.php @@ -0,0 +1,147 @@ +status, Status::Failed); + Assert::false($result->status->isSuccessful(), 'An aborted test must not let the run exit 0.'); + } + + /** + * The aborted tests are still counted and rendered alongside the surviving passing test. + */ + public function testLevelAbortIsCountedAndVisibleInTheSummary(): void + { + $result = self::runScenario('TestStageScenarios.php'); + + Assert::same($result->summary->total(), 3); + Assert::same($result->summary->count(Status::Passed), 1); + Assert::same($result->summary->count(Status::Aborted), 2); + + $statuses = []; + foreach ($result as $suite) { + foreach ($suite as $case) { + foreach ($case as $test) { + \assert($test instanceof TestResult); + $statuses[$test->info->name] = $test->status; + } + } + } + + Assert::same($statuses['throwsBeforeNext'] ?? null, Status::Aborted); + Assert::same($statuses['throwsAfterNext'] ?? null, Status::Aborted); + Assert::same($statuses['passesCleanly'] ?? null, Status::Passed); + } + + /** + * A throw before `$next()` at case level fails the run: none of the tests run and the + * suite is marked Error. + */ + public function caseLevelThrowBeforeNextFailsTheRun(): void + { + $result = self::runScenario('CaseBeforeScenario.php'); + + Assert::same($result->status, Status::Failed); + Assert::false($result->status->isSuccessful()); + Assert::same($result->summary->total(), 0); + } + + /** + * A throw after `$next()` at case level also fails the run, but the tests that already ran + * (and passed) vanish from the summary because the case result is discarded on unwind. + */ + public function caseLevelThrowAfterNextFailsTheRunButLosesResults(): void + { + $result = self::runScenario('CaseAfterScenario.php'); + + Assert::same($result->status, Status::Failed); + Assert::false($result->status->isSuccessful()); + Assert::same($result->summary->total(), 0); + } + + /** + * Sanity guard: without the interceptor exploding, the stub with clean/marked methods is not + * special — the message referenced here keeps the stubs and the interceptor in sync. + */ + public function interceptorMessageIsStable(): void + { + Assert::same(FailingInterceptor::MESSAGE, 'FailingInterceptor exploded'); + } + + /** + * Run a single stub file through a nested application and return the whole {@see RunResult}. + * + * No plugin is registered: {@see \Tests\Application\Stub\Pipeline\FailPipeline} self-binds + * {@see FailingInterceptor} via {@see \Testo\Pipeline\Attribute\FallbackInterceptor}, so the + * attribute alone drives the failure. + * + * @param non-empty-string $stubFile File name inside the Pipeline stub directory. + */ + private static function runScenario(string $stubFile): RunResult + { + $app = Application::createFromInput(); + $app->getContainer()->set( + new ApplicationConfig( + src: [], + suites: [ + new SuiteConfig( + name: 'PipelineFailure', + location: new FinderConfig(include: [self::STUB_DIR . '/' . $stubFile]), + ), + ], + ), + ApplicationConfig::class, + ); + + return $app->run(); + } +} diff --git a/tests/Application/Stub/Pipeline/CaseAfterScenario.php b/tests/Application/Stub/Pipeline/CaseAfterScenario.php new file mode 100644 index 00000000..8370e82d --- /dev/null +++ b/tests/Application/Stub/Pipeline/CaseAfterScenario.php @@ -0,0 +1,28 @@ +attribute->stage === FailStage::TestBefore and throw new \RuntimeException($this->message()); + + $result = $next($info); + + $this->attribute->stage === FailStage::TestAfter and throw new \RuntimeException($this->message()); + + return $result; + } + + #[\Override] + public function runTestCase(CaseInfo $info, callable $next): CaseResult + { + $this->attribute->stage === FailStage::CaseBefore and throw new \RuntimeException($this->message()); + + $result = $next($info); + + $this->attribute->stage === FailStage::CaseAfter and throw new \RuntimeException($this->message()); + + return $result; + } + + private function message(): string + { + return self::MESSAGE . ': ' . $this->attribute->stage->value; + } +} diff --git a/tests/Application/Stub/Pipeline/TestStageScenarios.php b/tests/Application/Stub/Pipeline/TestStageScenarios.php new file mode 100644 index 00000000..d86ce145 --- /dev/null +++ b/tests/Application/Stub/Pipeline/TestStageScenarios.php @@ -0,0 +1,36 @@ + Date: Fri, 3 Jul 2026 17:08:40 +0400 Subject: [PATCH 2/2] test(sandbox): add a pipeline-failure playground MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enables the sandbox suite in testo.php (skipped under TESTO_CI) and adds a sandbox file exercising the FailPipeline attribute at every stage — test-level and case-level throws before/after $next() — for eyeballing the terminal report. Assisted-By: Claude Opus 4.8 (1M context) --- testo.php | 16 ++-- .../Pipeline/PipelineFailureSandbox.php | 81 +++++++++++++++++++ 2 files changed, 89 insertions(+), 8 deletions(-) create mode 100644 tests/Sandbox/Pipeline/PipelineFailureSandbox.php diff --git a/testo.php b/testo.php index 52c3567d..697257d6 100644 --- a/testo.php +++ b/testo.php @@ -38,14 +38,14 @@ ), ], # If running in CI, skip the sandbox - // \filter_var(\getenv('TESTO_CI'), FILTER_VALIDATE_BOOLEAN) ? [] : [ - // new SuiteConfig( - // name: 'sandbox', - // location: new FinderConfig( - // include: ['tests/Sandbox'], - // ), - // ), - // ], + \filter_var(\getenv('TESTO_CI'), FILTER_VALIDATE_BOOLEAN) ? [] : [ + new SuiteConfig( + name: 'sandbox', + location: new FinderConfig( + include: ['tests/Sandbox'], + ), + ), + ], require 'bridge/mockery/tests/suites.php', require 'bridge/rector/tests/suites.php', require 'bridge/symfony-console/tests/suites.php', diff --git a/tests/Sandbox/Pipeline/PipelineFailureSandbox.php b/tests/Sandbox/Pipeline/PipelineFailureSandbox.php new file mode 100644 index 00000000..bf92937f --- /dev/null +++ b/tests/Sandbox/Pipeline/PipelineFailureSandbox.php @@ -0,0 +1,81 @@ +