fix(runner): fail the run when a test is aborted by a pipeline error

core/Application/Internal/Runner/CaseRunner.php

@@ -75,7 +75,12 @@ public function run(CaseInfo $info): CaseResult
                 );
 
                 $result = $this->testRunner->runTest($testInfo);
-                $result->status->isFailure() and $status = Status::Failed;
+                # A test aborted by a critical interceptor failure has an unknown verdict — treat it as
+                # a case failure so it propagates up to the suite/run status and the process exit code
+                # (every reporter already renders an abort as a failure). Deliberately broader than
+                # Status::isFailure(), which must keep Aborted out so retry/repeat don't retry an abort.
+                ($result->status->isFailure() || $result->status === Status::Aborted)
+                    and $status = Status::Failed;
 
                 $results[] = $result;
             } catch (\Throwable $throwable) {

core/Output/Terminal/Renderer/DotSymbol.php

@@ -14,4 +14,5 @@ enum DotSymbol: string
     case Skipped = '-';
     case Risky = 'R';
     case Error = 'E';
+    case Aborted = 'A';
 }

core/Output/Terminal/Renderer/Formatter.php

@@ -404,7 +404,8 @@ private static function formatDotRun(FormattedItem $item): string
             Status::Passed => DotSymbol::Passed->value,
             Status::Failed => Style::error(DotSymbol::Failed->value),
             Status::Skipped => Style::warning(DotSymbol::Skipped->value),
-            Status::Error, Status::Aborted => Style::error(DotSymbol::Error->value),
+            Status::Error => Style::error(DotSymbol::Error->value),
+            Status::Aborted => Style::error(DotSymbol::Aborted->value),
             Status::Risky => Style::warning(DotSymbol::Risky->value),
             Status::Flaky => Style::info(DotSymbol::Passed->value),
             Status::Cancelled => Style::dim(DotSymbol::Skipped->value),
@@ -422,7 +423,8 @@ private static function getStatusSymbol(Status $status): string
             Status::Passed => Style::success(Symbol::Success->value),
             Status::Failed => Style::error(Symbol::Failure->value),
             Status::Skipped => Style::warning(Symbol::Skipped->value),
-            Status::Error, Status::Aborted => Style::error(Symbol::Error->value),
+            Status::Error => Style::error(Symbol::Error->value),
+            Status::Aborted => Style::error(Symbol::Aborted->value),
             Status::Risky => Style::warning(Symbol::Risky->value),
             Status::Flaky => Style::info(Symbol::Flaky->value),
             Status::Cancelled => Style::dim(Symbol::Skipped->value),

core/Output/Terminal/Renderer/Symbol.php

@@ -15,5 +15,6 @@ enum Symbol: string
     case Risky = '?';
     case Flaky = '~';
     case Error = 'E';
+    case Aborted = 'A';
     case DataProvider = '◆';
 }

testo.php

@@ -38,14 +38,14 @@
             ),
         ],
         # If running in CI, skip the sandbox
-        // \filter_var(\getenv('TESTO_CI'), FILTER_VALIDATE_BOOLEAN) ? [] : [
-        //     new SuiteConfig(
-        //         name: 'sandbox',
-        //         location: new FinderConfig(
-        //             include: ['tests/Sandbox'],
-        //         ),
-        //     ),
-        // ],
+        \filter_var(\getenv('TESTO_CI'), FILTER_VALIDATE_BOOLEAN) ? [] : [
+            new SuiteConfig(
+                name: 'sandbox',
+                location: new FinderConfig(
+                    include: ['tests/Sandbox'],
+                ),
+            ),
+        ],
         require 'bridge/mockery/tests/suites.php',
         require 'bridge/rector/tests/suites.php',
         require 'bridge/symfony-console/tests/suites.php',

tests/Application/Feature/Runner/PipelineFailureStatusTest.php

@@ -0,0 +1,147 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Application\Feature\Runner;
+
+use Testo\Application\Application;
+use Testo\Application\Config\ApplicationConfig;
+use Testo\Application\Config\FinderConfig;
+use Testo\Application\Config\SuiteConfig;
+use Testo\Application\Internal\Runner\CaseRunner;
+use Testo\Application\Internal\Runner\SuiteRunner;
+use Testo\Application\Internal\Runner\TestRunner;
+use Testo\Assert;
+use Testo\Codecov\Covers;
+use Testo\Core\Context\RunResult;
+use Testo\Core\Context\TestResult;
+use Testo\Core\Value\Status;
+use Testo\Test;
+use Tests\Application\Stub\Pipeline\FailingInterceptor;
+
+/**
+ * Pins how a throw *from inside an interceptor* (not from the test body) is surfaced
+ * by the runners, depending on the pipeline stage it happens at.
+ *
+ * Findings this test locks in:
+ *
+ *  - A throw in {@see \Testo\Pipeline\Middleware\TestRunInterceptor::runTest()} (before or after
+ *    `$next()`) is caught by {@see TestRunner} and turned into a {@see Status::Aborted} test result.
+ *    {@see Status::isFailure()} stays `false` for `Aborted` (so retry/repeat never retry an abort),
+ *    but {@see CaseRunner} escalates the case to {@see Status::Failed} on an aborted test, so the
+ *    suite/run status and the process exit code agree with the "FAILED" the reporters render.
+ *
+ *  - A throw in {@see \Testo\Pipeline\Middleware\TestCaseRunInterceptor::runTestCase()} escapes the
+ *    case pipeline, is caught by {@see SuiteRunner} and marks the suite {@see Status::Error}, which
+ *    *does* fail the run — but the individual test results are dropped from the summary.
+ *
+ * @see \Testo\Core\Value\Status::Aborted
+ */
+#[Test]
+#[Covers(TestRunner::class)]
+#[Covers(SuiteRunner::class)]
+final class PipelineFailureStatusTest
+{
+    private const STUB_DIR = __DIR__ . '/../../Stub/Pipeline';
+
+    /**
+     * A broken test-level interceptor aborts two tests; the run must fail (exit code 1) so it
+     * agrees with the "FAILED" banner the reporters print, instead of silently exiting 0.
+     */
+    public function testLevelAbortFailsTheRun(): void
+    {
+        $result = self::runScenario('TestStageScenarios.php');
+
+        Assert::same($result->status, Status::Failed);
+        Assert::false($result->status->isSuccessful(), 'An aborted test must not let the run exit 0.');
+    }
+
+    /**
+     * The aborted tests are still counted and rendered alongside the surviving passing test.
+     */
+    public function testLevelAbortIsCountedAndVisibleInTheSummary(): void
+    {
+        $result = self::runScenario('TestStageScenarios.php');
+
+        Assert::same($result->summary->total(), 3);
+        Assert::same($result->summary->count(Status::Passed), 1);
+        Assert::same($result->summary->count(Status::Aborted), 2);
+
+        $statuses = [];
+        foreach ($result as $suite) {
+            foreach ($suite as $case) {
+                foreach ($case as $test) {
+                    \assert($test instanceof TestResult);
+                    $statuses[$test->info->name] = $test->status;
+                }
+            }
+        }
+
+        Assert::same($statuses['throwsBeforeNext'] ?? null, Status::Aborted);
+        Assert::same($statuses['throwsAfterNext'] ?? null, Status::Aborted);
+        Assert::same($statuses['passesCleanly'] ?? null, Status::Passed);
+    }
+
+    /**
+     * A throw before `$next()` at case level fails the run: none of the tests run and the
+     * suite is marked Error.
+     */
+    public function caseLevelThrowBeforeNextFailsTheRun(): void
+    {
+        $result = self::runScenario('CaseBeforeScenario.php');
+
+        Assert::same($result->status, Status::Failed);
+        Assert::false($result->status->isSuccessful());
+        Assert::same($result->summary->total(), 0);
+    }
+
+    /**
+     * A throw after `$next()` at case level also fails the run, but the tests that already ran
+     * (and passed) vanish from the summary because the case result is discarded on unwind.
+     */
+    public function caseLevelThrowAfterNextFailsTheRunButLosesResults(): void
+    {
+        $result = self::runScenario('CaseAfterScenario.php');
+
+        Assert::same($result->status, Status::Failed);
+        Assert::false($result->status->isSuccessful());
+        Assert::same($result->summary->total(), 0);
+    }
+
+    /**
+     * Sanity guard: without the interceptor exploding, the stub with clean/marked methods is not
+     * special — the message referenced here keeps the stubs and the interceptor in sync.
+     */
+    public function interceptorMessageIsStable(): void
+    {
+        Assert::same(FailingInterceptor::MESSAGE, 'FailingInterceptor exploded');
+    }
+
+    /**
+     * Run a single stub file through a nested application and return the whole {@see RunResult}.
+     *
+     * No plugin is registered: {@see \Tests\Application\Stub\Pipeline\FailPipeline} self-binds
+     * {@see FailingInterceptor} via {@see \Testo\Pipeline\Attribute\FallbackInterceptor}, so the
+     * attribute alone drives the failure.
+     *
+     * @param non-empty-string $stubFile File name inside the Pipeline stub directory.
+     */
+    private static function runScenario(string $stubFile): RunResult
+    {
+        $app = Application::createFromInput();
+        $app->getContainer()->set(
+            new ApplicationConfig(
+                src: [],
+                suites: [
+                    new SuiteConfig(
+                        name: 'PipelineFailure',
+                        location: new FinderConfig(include: [self::STUB_DIR . '/' . $stubFile]),
+                    ),
+                ],
+            ),
+            ApplicationConfig::class,
+        );
+
+        return $app->run();
+    }
+}

tests/Application/Stub/Pipeline/CaseAfterScenario.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Application\Stub\Pipeline;
+
+use Testo\Assert;
+use Testo\Test;
+
+/**
+ * Case-level scenario: the interceptor throws in `runTestCase` AFTER `$next()`,
+ * so the tests below actually run (and pass) but the resulting {@see CaseResult}
+ * is discarded when the throw unwinds the case pipeline.
+ */
+#[Test]
+#[FailPipeline(FailStage::CaseAfter)]
+final class CaseAfterScenario
+{
+    public function runsButResultIsLostA(): void
+    {
+        Assert::same(1, 1);
+    }
+
+    public function runsButResultIsLostB(): void
+    {
+        Assert::same(1, 1);
+    }
+}

tests/Application/Stub/Pipeline/CaseBeforeScenario.php

@@ -0,0 +1,27 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Application\Stub\Pipeline;
+
+use Testo\Assert;
+use Testo\Test;
+
+/**
+ * Case-level scenario: the interceptor throws in `runTestCase` BEFORE `$next()`,
+ * so none of the tests below ever execute.
+ */
+#[Test]
+#[FailPipeline(FailStage::CaseBefore)]
+final class CaseBeforeScenario
+{
+    public function neverRunsA(): void
+    {
+        Assert::same(1, 1);
+    }
+
+    public function neverRunsB(): void
+    {
+        Assert::same(1, 1);
+    }
+}

tests/Application/Stub/Pipeline/FailPipeline.php

@@ -0,0 +1,29 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Application\Stub\Pipeline;
+
+use Testo\Pipeline\Attribute\FallbackInterceptor;
+use Testo\Pipeline\Attribute\Interceptable;
+
+/**
+ * Marks a test method (or a whole test case) so that {@see FailingInterceptor}
+ * throws at the configured {@see FailStage}.
+ *
+ * The attribute is self-binding: {@see FallbackInterceptor} points the pipeline's
+ * {@see \Testo\Pipeline\Internal\AttributesInterceptor} at {@see FailingInterceptor}, so the
+ * interceptor is attached wherever this attribute appears — no plugin registration required.
+ *
+ * Placed on a method → drives the test-level stages ({@see FailStage::TestBefore},
+ * {@see FailStage::TestAfter}); placed on a class → drives the case-level stages
+ * ({@see FailStage::CaseBefore}, {@see FailStage::CaseAfter}).
+ */
+#[\Attribute(\Attribute::TARGET_METHOD | \Attribute::TARGET_CLASS)]
+#[FallbackInterceptor(FailingInterceptor::class)]
+final readonly class FailPipeline implements Interceptable
+{
+    public function __construct(
+        public FailStage $stage,
+    ) {}
+}

tests/Application/Stub/Pipeline/FailStage.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Application\Stub\Pipeline;
+
+/**
+ * The pipeline stage at which {@see FailingInterceptor} should throw.
+ *
+ * Each case reproduces one place where a user-authored interceptor can blow up:
+ * before or after the `$next()` call, at either the test or the test-case level.
+ */
+enum FailStage: string
+{
+    /** Throw in {@see FailingInterceptor::runTest()} before calling `$next()`. */
+    case TestBefore = 'test-before';
+
+    /** Throw in {@see FailingInterceptor::runTest()} after `$next()` returned. */
+    case TestAfter = 'test-after';
+
+    /** Throw in {@see FailingInterceptor::runTestCase()} before calling `$next()`. */
+    case CaseBefore = 'case-before';
+
+    /** Throw in {@see FailingInterceptor::runTestCase()} after `$next()` returned. */
+    case CaseAfter = 'case-after';
+}

tests/Application/Stub/Pipeline/FailingInterceptor.php

@@ -0,0 +1,65 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Application\Stub\Pipeline;
+
+use Testo\Core\Context\CaseInfo;
+use Testo\Core\Context\CaseResult;
+use Testo\Core\Context\TestInfo;
+use Testo\Core\Context\TestResult;
+use Testo\Pipeline\Attribute\InterceptorOptions;
+use Testo\Pipeline\Middleware\TestCaseRunInterceptor;
+use Testo\Pipeline\Middleware\TestRunInterceptor;
+
+/**
+ * A deliberately misbehaving user-style interceptor: it throws a plain {@see \RuntimeException}
+ * at the {@see FailStage} declared by the {@see FailPipeline} attribute it was attached for.
+ *
+ * The interceptor is wired to the attribute via {@see \Testo\Pipeline\Attribute\FallbackInterceptor}
+ * on {@see FailPipeline}, so {@see \Testo\Pipeline\Internal\AttributesInterceptor} attaches it (and
+ * injects the attribute instance here) only for marked tests/cases — no plugin needed. Method-level
+ * marks reach {@see self::runTest()}; class-level marks reach both, but each method reacts only to
+ * its own stages.
+ *
+ * The point is to observe how a throw from inside the pipeline (as opposed to a throw from the test
+ * body) is reflected in the resulting statuses and the final report.
+ */
+#[InterceptorOptions(order: InterceptorOptions::ORDER_CLOSE_TO_TEST)]
+final readonly class FailingInterceptor implements TestRunInterceptor, TestCaseRunInterceptor
+{
+    public const MESSAGE = 'FailingInterceptor exploded';
+
+    public function __construct(
+        private FailPipeline $attribute,
+    ) {}
+
+    #[\Override]
+    public function runTest(TestInfo $info, callable $next): TestResult
+    {
+        $this->attribute->stage === FailStage::TestBefore and throw new \RuntimeException($this->message());
+
+        $result = $next($info);
+
+        $this->attribute->stage === FailStage::TestAfter and throw new \RuntimeException($this->message());
+
+        return $result;
+    }
+
+    #[\Override]
+    public function runTestCase(CaseInfo $info, callable $next): CaseResult
+    {
+        $this->attribute->stage === FailStage::CaseBefore and throw new \RuntimeException($this->message());
+
+        $result = $next($info);
+
+        $this->attribute->stage === FailStage::CaseAfter and throw new \RuntimeException($this->message());
+
+        return $result;
+    }
+
+    private function message(): string
+    {
+        return self::MESSAGE . ': ' . $this->attribute->stage->value;
+    }
+}