diff --git a/core/Application/Internal/Runner/CaseRunner.php b/core/Application/Internal/Runner/CaseRunner.php index fe3d8e09..c8497247 100644 --- a/core/Application/Internal/Runner/CaseRunner.php +++ b/core/Application/Internal/Runner/CaseRunner.php @@ -75,7 +75,12 @@ public function run(CaseInfo $info): CaseResult ); $result = $this->testRunner->runTest($testInfo); - $result->status->isFailure() and $status = Status::Failed; + # A test aborted by a critical interceptor failure has an unknown verdict — treat it as + # a case failure so it propagates up to the suite/run status and the process exit code + # (every reporter already renders an abort as a failure). Deliberately broader than + # Status::isFailure(), which must keep Aborted out so retry/repeat don't retry an abort. + ($result->status->isFailure() || $result->status === Status::Aborted) + and $status = Status::Failed; $results[] = $result; } catch (\Throwable $throwable) { diff --git a/core/Output/Terminal/Renderer/DotSymbol.php b/core/Output/Terminal/Renderer/DotSymbol.php index c3afff18..57ebf5f4 100644 --- a/core/Output/Terminal/Renderer/DotSymbol.php +++ b/core/Output/Terminal/Renderer/DotSymbol.php @@ -14,4 +14,5 @@ enum DotSymbol: string case Skipped = '-'; case Risky = 'R'; case Error = 'E'; + case Aborted = 'A'; } diff --git a/core/Output/Terminal/Renderer/Formatter.php b/core/Output/Terminal/Renderer/Formatter.php index 12a8389d..5917715a 100644 --- a/core/Output/Terminal/Renderer/Formatter.php +++ b/core/Output/Terminal/Renderer/Formatter.php @@ -404,7 +404,8 @@ private static function formatDotRun(FormattedItem $item): string Status::Passed => DotSymbol::Passed->value, Status::Failed => Style::error(DotSymbol::Failed->value), Status::Skipped => Style::warning(DotSymbol::Skipped->value), - Status::Error, Status::Aborted => Style::error(DotSymbol::Error->value), + Status::Error => Style::error(DotSymbol::Error->value), + Status::Aborted => Style::error(DotSymbol::Aborted->value), Status::Risky => Style::warning(DotSymbol::Risky->value), Status::Flaky => Style::info(DotSymbol::Passed->value), Status::Cancelled => Style::dim(DotSymbol::Skipped->value), @@ -422,7 +423,8 @@ private static function getStatusSymbol(Status $status): string Status::Passed => Style::success(Symbol::Success->value), Status::Failed => Style::error(Symbol::Failure->value), Status::Skipped => Style::warning(Symbol::Skipped->value), - Status::Error, Status::Aborted => Style::error(Symbol::Error->value), + Status::Error => Style::error(Symbol::Error->value), + Status::Aborted => Style::error(Symbol::Aborted->value), Status::Risky => Style::warning(Symbol::Risky->value), Status::Flaky => Style::info(Symbol::Flaky->value), Status::Cancelled => Style::dim(Symbol::Skipped->value), diff --git a/core/Output/Terminal/Renderer/Symbol.php b/core/Output/Terminal/Renderer/Symbol.php index 892bbf8b..92eb4b54 100644 --- a/core/Output/Terminal/Renderer/Symbol.php +++ b/core/Output/Terminal/Renderer/Symbol.php @@ -15,5 +15,6 @@ enum Symbol: string case Risky = '?'; case Flaky = '~'; case Error = 'E'; + case Aborted = 'A'; case DataProvider = '◆'; } diff --git a/testo.php b/testo.php index 52c3567d..697257d6 100644 --- a/testo.php +++ b/testo.php @@ -38,14 +38,14 @@ ), ], # If running in CI, skip the sandbox - // \filter_var(\getenv('TESTO_CI'), FILTER_VALIDATE_BOOLEAN) ? [] : [ - // new SuiteConfig( - // name: 'sandbox', - // location: new FinderConfig( - // include: ['tests/Sandbox'], - // ), - // ), - // ], + \filter_var(\getenv('TESTO_CI'), FILTER_VALIDATE_BOOLEAN) ? [] : [ + new SuiteConfig( + name: 'sandbox', + location: new FinderConfig( + include: ['tests/Sandbox'], + ), + ), + ], require 'bridge/mockery/tests/suites.php', require 'bridge/rector/tests/suites.php', require 'bridge/symfony-console/tests/suites.php', diff --git a/tests/Application/Feature/Runner/PipelineFailureStatusTest.php b/tests/Application/Feature/Runner/PipelineFailureStatusTest.php new file mode 100644 index 00000000..7845c1c2 --- /dev/null +++ b/tests/Application/Feature/Runner/PipelineFailureStatusTest.php @@ -0,0 +1,147 @@ +status, Status::Failed); + Assert::false($result->status->isSuccessful(), 'An aborted test must not let the run exit 0.'); + } + + /** + * The aborted tests are still counted and rendered alongside the surviving passing test. + */ + public function testLevelAbortIsCountedAndVisibleInTheSummary(): void + { + $result = self::runScenario('TestStageScenarios.php'); + + Assert::same($result->summary->total(), 3); + Assert::same($result->summary->count(Status::Passed), 1); + Assert::same($result->summary->count(Status::Aborted), 2); + + $statuses = []; + foreach ($result as $suite) { + foreach ($suite as $case) { + foreach ($case as $test) { + \assert($test instanceof TestResult); + $statuses[$test->info->name] = $test->status; + } + } + } + + Assert::same($statuses['throwsBeforeNext'] ?? null, Status::Aborted); + Assert::same($statuses['throwsAfterNext'] ?? null, Status::Aborted); + Assert::same($statuses['passesCleanly'] ?? null, Status::Passed); + } + + /** + * A throw before `$next()` at case level fails the run: none of the tests run and the + * suite is marked Error. + */ + public function caseLevelThrowBeforeNextFailsTheRun(): void + { + $result = self::runScenario('CaseBeforeScenario.php'); + + Assert::same($result->status, Status::Failed); + Assert::false($result->status->isSuccessful()); + Assert::same($result->summary->total(), 0); + } + + /** + * A throw after `$next()` at case level also fails the run, but the tests that already ran + * (and passed) vanish from the summary because the case result is discarded on unwind. + */ + public function caseLevelThrowAfterNextFailsTheRunButLosesResults(): void + { + $result = self::runScenario('CaseAfterScenario.php'); + + Assert::same($result->status, Status::Failed); + Assert::false($result->status->isSuccessful()); + Assert::same($result->summary->total(), 0); + } + + /** + * Sanity guard: without the interceptor exploding, the stub with clean/marked methods is not + * special — the message referenced here keeps the stubs and the interceptor in sync. + */ + public function interceptorMessageIsStable(): void + { + Assert::same(FailingInterceptor::MESSAGE, 'FailingInterceptor exploded'); + } + + /** + * Run a single stub file through a nested application and return the whole {@see RunResult}. + * + * No plugin is registered: {@see \Tests\Application\Stub\Pipeline\FailPipeline} self-binds + * {@see FailingInterceptor} via {@see \Testo\Pipeline\Attribute\FallbackInterceptor}, so the + * attribute alone drives the failure. + * + * @param non-empty-string $stubFile File name inside the Pipeline stub directory. + */ + private static function runScenario(string $stubFile): RunResult + { + $app = Application::createFromInput(); + $app->getContainer()->set( + new ApplicationConfig( + src: [], + suites: [ + new SuiteConfig( + name: 'PipelineFailure', + location: new FinderConfig(include: [self::STUB_DIR . '/' . $stubFile]), + ), + ], + ), + ApplicationConfig::class, + ); + + return $app->run(); + } +} diff --git a/tests/Application/Stub/Pipeline/CaseAfterScenario.php b/tests/Application/Stub/Pipeline/CaseAfterScenario.php new file mode 100644 index 00000000..8370e82d --- /dev/null +++ b/tests/Application/Stub/Pipeline/CaseAfterScenario.php @@ -0,0 +1,28 @@ +attribute->stage === FailStage::TestBefore and throw new \RuntimeException($this->message()); + + $result = $next($info); + + $this->attribute->stage === FailStage::TestAfter and throw new \RuntimeException($this->message()); + + return $result; + } + + #[\Override] + public function runTestCase(CaseInfo $info, callable $next): CaseResult + { + $this->attribute->stage === FailStage::CaseBefore and throw new \RuntimeException($this->message()); + + $result = $next($info); + + $this->attribute->stage === FailStage::CaseAfter and throw new \RuntimeException($this->message()); + + return $result; + } + + private function message(): string + { + return self::MESSAGE . ': ' . $this->attribute->stage->value; + } +} diff --git a/tests/Application/Stub/Pipeline/TestStageScenarios.php b/tests/Application/Stub/Pipeline/TestStageScenarios.php new file mode 100644 index 00000000..d86ce145 --- /dev/null +++ b/tests/Application/Stub/Pipeline/TestStageScenarios.php @@ -0,0 +1,36 @@ +