test_runner: support runner bailout at files level

Author: pmarchini

doc/api/cli.md

@@ -2614,6 +2614,21 @@ Starts the Node.js command line test runner. This flag cannot be combined with
 See the documentation on [running tests from the command line][]
 for more details.
 
+### `--test-bail`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+Stops the test runner after the first test failure.
+
+Behavior depends on `--test-isolation`.
+See the [test runner execution model][] section for details.
+
+This flag cannot be combined with `--watch`.
+
 ### `--test-concurrency`
 
 <!-- YAML
@@ -3679,6 +3694,7 @@ one is included in the list below.
 * `--secure-heap-min`
 * `--secure-heap`
 * `--snapshot-blob`
+* `--test-bail`
 * `--test-coverage-branches`
 * `--test-coverage-exclude`
 * `--test-coverage-functions`

doc/api/test.md

@@ -611,6 +611,16 @@ each other in ways that are not possible when isolation is enabled. For example,
 if a test relies on global state, it is possible for that state to be modified
 by a test originating from another file.
 
+When bail is enabled (via [`--test-bail`][] or `run({ bail: true })`),
+isolation mode changes how execution stops:
+
+* In `'process'` isolation, no new test files are started after the first
+  failure, and test files that have already started are not forcibly
+  terminated. Within already-started files, tests that have not started yet
+  may still be aborted by bailout.
+* In `'none'` isolation, no new tests are started and queued tests are
+  cancelled.
+
 #### Child process option inheritance
 
 When running tests in process isolation mode (the default), spawned child processes
@@ -1505,6 +1515,15 @@ changes:
   * `argv` {Array} An array of CLI flags to pass to each test file when spawning the
     subprocesses. This option has no effect when `isolation` is `'none'`.
     **Default:** `[]`.
+  * `bail` {boolean} Stops the test run after the first failure.
+    If `isolation` is `'process'`, no new test files are started after the
+    first failure, and files that have already started are not forcibly
+    terminated. Within already-started files, tests that have not started yet
+    may still be aborted by bailout.
+    If `isolation` is `'none'`, no new tests are started and queued tests are
+    cancelled.
+    This option cannot be used together with `watch`.
+    **Default:** `false`.
   * `signal` {AbortSignal} Allows aborting an in-progress test execution.
   * `testNamePatterns` {string|RegExp|Array} A String, RegExp or a RegExp Array,
     that can be used to only run tests whose name matches the provided pattern.
@@ -3232,6 +3251,22 @@ are defined, while others are emitted in the order that the tests execute.
 
 Emitted when code coverage is enabled and all tests have completed.
 
+### Event: `'test:bail'`
+
+* `data` {Object}
+  * `column` {number|undefined} The column number where the bailout originated,
+    or `undefined` if it was run through the REPL.
+  * `file` {string|undefined} The path of the test file, `undefined` if test
+    was run through the REPL.
+  * `line` {number|undefined} The line number where the bailout originated, or
+    `undefined` if it was run through the REPL.
+  * `nesting` {number} The nesting level of the test.
+  * `test` {string} The bailout message.
+
+Emitted when bail is enabled and the first failure triggers bailout behavior.
+In `'process'` isolation this means no new test files are started, while in
+`'none'` isolation no new tests are started and queued tests are cancelled.
+
 ### Event: `'test:complete'`
 
 * `data` {Object}
@@ -4096,6 +4131,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`--experimental-test-module-mocks`]: cli.md#--experimental-test-module-mocks
 [`--import`]: cli.md#--importmodule
 [`--no-strip-types`]: cli.md#--no-strip-types
+[`--test-bail`]: cli.md#--test-bail
 [`--test-concurrency`]: cli.md#--test-concurrency
 [`--test-coverage-exclude`]: cli.md#--test-coverage-exclude
 [`--test-coverage-include`]: cli.md#--test-coverage-include

doc/node-config-schema.json

@@ -671,6 +671,9 @@
         "test": {
           "type": "boolean"
         },
+        "test-bail": {
+          "type": "boolean"
+        },
         "test-concurrency": {
           "type": "number"
         },

doc/node.1

@@ -471,6 +471,15 @@ Specify the minimum allocation from the OpenSSL secure heap. The default is 2. T
 .It Fl -test
 Starts the Node.js command line test runner.
 .
+.It Fl -test-bail
+Stops the test runner after the first test failure.
+This option is in early development.
+Behavior depends on the selected
+.Fl -test-isolation
+mode.
+This flag cannot be combined with
+.Fl -watch .
+.
 .It Fl -test-concurrency
 The maximum number of test files that the test runner CLI will execute
 concurrently.

lib/internal/test_runner/harness.js

@@ -52,6 +52,8 @@ function createTestTree(rootTestOptions, globalOptions) {
     buildSuites: [],
     isWaitingForBuildPhase: false,
     watching: false,
+    bail: globalOptions.bail,
+    bailedOut: false,
     config: globalOptions,
     coverage: null,
     resetCounters() {

lib/internal/test_runner/reporter/spec.js

@@ -11,7 +11,7 @@ const {
 const assert = require('assert');
 const Transform = require('internal/streams/transform');
 const colors = require('internal/util/colors');
-const { kSubtestsFailed } = require('internal/test_runner/test');
+const { kSubtestsFailed, kBailedOut } = require('internal/test_runner/test');
 const { getCoverageReport } = require('internal/test_runner/utils');
 const { relative } = require('path');
 const {
@@ -78,8 +78,10 @@ class SpecReporter extends Transform {
   }
   #handleEvent({ type, data }) {
     switch (type) {
+      case 'test:bail':
+        return `${reporterColorMap['test:bail']}${reporterUnicodeSymbolMap[type]}Bailing out!${colors.white}\n`;
       case 'test:fail':
-        if (data.details?.error?.failureType !== kSubtestsFailed) {
+        if (data.details?.error?.failureType !== kSubtestsFailed && data.details?.error?.failureType !== kBailedOut) {
           ArrayPrototypePush(this.#failedTests, data);
         }
         return this.#handleTestReportEvent(type, data);

lib/internal/test_runner/reporter/utils.js

@@ -24,6 +24,7 @@ const reporterUnicodeSymbolMap = {
   'test:coverage': '\u2139 ',
   'arrow:right': '\u25B6 ',
   'hyphen:minus': '\uFE63 ',
+  'test:bail': '\u26A0 ',
 };
 
 const reporterColorMap = {
@@ -37,6 +38,9 @@ const reporterColorMap = {
   get 'test:diagnostic'() {
     return colors.blue;
   },
+  get 'test:bail'() {
+    return colors.yellow;
+  },
   get 'info'() {
     return colors.blue;
   },

lib/internal/test_runner/runner.js

@@ -22,6 +22,8 @@ const {
   SafePromiseAll,
   SafePromiseAllReturnVoid,
   SafePromiseAllSettledReturnVoid,
+  SafePromisePrototypeFinally,
+  SafePromiseRace,
   SafeSet,
   StringPrototypeIndexOf,
   StringPrototypeSlice,
@@ -147,6 +149,7 @@ function getRunArgs(path, { forceExit,
                             testNamePatterns,
                             testSkipPatterns,
                             only,
+                            bail,
                             argv: suppliedArgs,
                             execArgv,
                             rerunFailuresFilePath,
@@ -185,6 +188,9 @@ function getRunArgs(path, { forceExit,
   if (only === true) {
     ArrayPrototypePush(runArgs, '--test-only');
   }
+  if (bail === true) {
+    ArrayPrototypePush(runArgs, '--test-bail');
+  }
   if (timeout != null) {
     ArrayPrototypePush(runArgs, `--test-timeout=${timeout}`);
   }
@@ -271,9 +277,13 @@ class FileTest extends Test {
     this.reporter[kEmitMessage](item.type, item.data);
   }
   #accumulateReportItem(item) {
-    if (item.type !== 'test:pass' && item.type !== 'test:fail') {
+    if (item.type !== 'test:pass' && item.type !== 'test:fail' && item.type !== 'test:bail') {
       return;
     }
+    // If a test failure occurred and bail is enabled, emit a bail event after reporting the failure
+    if (item.type === 'test:bail' && this.root.harness?.bail && !this.root.harness.bailedOut) {
+      this.root.harness.bailedOut = true;
+    }
     this.#reportedChildren++;
     if (item.data.nesting === 0 && item.type === 'test:fail') {
       this.failedSubtests = true;
@@ -604,6 +614,7 @@ function run(options = kEmptyObject) {
   } = options;
   const {
     concurrency,
+    bail,
     timeout,
     signal,
     files,
@@ -747,7 +758,9 @@ function run(options = kEmptyObject) {
     functionCoverage: functionCoverage,
     cwd,
     globalSetupPath,
+    bail,
   };
+  
   const root = createTestTree(rootTestOptions, globalOptions);
   let testFiles = files ?? createTestFileList(globPatterns, cwd);
   const { isTestRunner } = globalOptions;
@@ -756,10 +769,18 @@ function run(options = kEmptyObject) {
     testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1);
   }
 
+  if (bail) {
+    validateBoolean(bail, 'options.bail');
+    if (watch) {
+      throw new ERR_INVALID_ARG_VALUE('options.bail', watch, 'bail not supported with watch mode');
+    }
+  }
+
   let teardown;
   let postRun;
   let filesWatcher;
   let runFiles;
+
   const opts = {
     __proto__: null,
     root,
@@ -770,6 +791,7 @@ function run(options = kEmptyObject) {
     hasFiles: files != null,
     globPatterns,
     only,
+    bail,
     forceExit,
     cwd,
     isolation,
@@ -792,15 +814,53 @@ function run(options = kEmptyObject) {
       teardown = () => root.harness.teardown();
     }
 
-    runFiles = () => {
-      root.harness.bootstrapPromise = null;
-      root.harness.buildPromise = null;
-      return SafePromiseAllSettledReturnVoid(testFiles, (path) => {
-        const subtest = runTestFile(path, filesWatcher, opts);
-        filesWatcher?.runningSubtests.set(path, subtest);
-        return subtest;
-      });
-    };
+    if (bail) {
+      runFiles = async () => {
+        root.harness.bootstrapPromise = null;
+        root.harness.buildPromise = null;
+
+        const running = new SafeSet();
+        let index = 0;
+
+        const shouldBail = () => bail && root.harness.bailedOut;
+
+        const enqueueNext = () => {
+          if (index < testFiles.length && !shouldBail()) {
+            const path = testFiles[index++];
+            const subtest = runTestFile(path, filesWatcher, opts);
+            filesWatcher?.runningSubtests.set(path, subtest);
+            running.add(subtest);
+            SafePromisePrototypeFinally(subtest, () => running.delete(subtest));
+          }
+        };
+
+        // Fill initial pool up to root test concurrency
+        // We use root test concurrency here because concurrency logic is handled at test level.
+        while (running.size < root.concurrency && index < testFiles.length && !shouldBail()) {
+          enqueueNext();
+        }
+
+        // As each test completes, enqueue the next one
+        while (running.size > 0) {
+          await SafePromiseRace([...running]);
+
+          // Refill pool after completion(s)
+          while (running.size < root.concurrency && index < testFiles.length && !shouldBail()) {
+            enqueueNext();
+          }
+        }
+      };
+    } else {
+      runFiles = () => {
+        root.harness.bootstrapPromise = null;
+        root.harness.buildPromise = null;
+        return SafePromiseAllSettledReturnVoid(testFiles, (path) => {
+          const subtest = runTestFile(path, filesWatcher, opts);
+          filesWatcher?.runningSubtests.set(path, subtest);
+          return subtest;
+        });
+      };
+    }
   } else if (isolation === 'none') {
     if (watch) {
       const absoluteTestFiles = ArrayPrototypeMap(testFiles, (file) => (isAbsolute(file) ? file : resolve(cwd, file)));

lib/internal/test_runner/test.js

@@ -78,6 +78,7 @@ const { bigint: hrtime } = process.hrtime;
 const kCallbackAndPromisePresent = 'callbackAndPromisePresent';
 const kCancelledByParent = 'cancelledByParent';
 const kAborted = 'testAborted';
+const kBailedOut = 'bailedOut';
 const kParentAlreadyFinished = 'parentAlreadyFinished';
 const kSubtestsFailed = 'subtestsFailed';
 const kTestCodeFailure = 'testCodeFailure';
@@ -580,7 +581,7 @@ class Test extends AsyncResource {
       }
     }
 
-    switch (typeof concurrency) {
+    switch (typeof concurrency) { // <-- here we are overriding this.concurrency with the value from options!
       case 'number':
         validateUint32(concurrency, 'options.concurrency', true);
         this.concurrency = concurrency;
@@ -780,6 +781,10 @@ class Test extends AsyncResource {
    */
   async processPendingSubtests() {
     while (this.pendingSubtests.length > 0 && this.hasConcurrency()) {
+      if (this.root.harness?.bailedOut) {
+        queueMicrotask( () => this.postRun(new ERR_TEST_FAILURE("Test was aborted due to bailout", kBailedOut)));
+        break;
+      }
       const deferred = ArrayPrototypeShift(this.pendingSubtests);
       const test = deferred.test;
       test.reporter.dequeue(test.nesting, test.loc, test.name, this.reportedType);
@@ -1382,6 +1387,10 @@ class Test extends AsyncResource {
       this.reporter.ok(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive);
     } else {
       this.reporter.fail(this.nesting, this.loc, this.testNumber, this.name, report.details, report.directive);
+      if (this.root.harness?.bail && !this.root.harness.bailedOut) {
+        this.reporter.bail(this.nesting, this.loc, 'bailing out due to test failure');
+        this.root.harness.bailedOut = true;
+      }
     }
 
     for (let i = 0; i < this.diagnostics.length; i++) {
@@ -1558,6 +1567,7 @@ module.exports = {
   kTestCodeFailure,
   kTestTimeoutFailure,
   kAborted,
+  kBailedOut,
   kUnwrapErrors,
   Suite,
   Test,

lib/internal/test_runner/tests_stream.js

@@ -149,6 +149,15 @@ class TestsStream extends Readable {
     });
   }
 
+  bail(nesting, loc, test) {
+    this[kEmitMessage]('test:bail', {
+      __proto__: null,
+      nesting,
+      test,
+      ...loc,
+    });
+  }
+
   end() {
     this.#tryPush(null);
   }