diff --git a/apps/rush/package.json b/apps/rush/package.json index 692fd79d614..d09c7f90eb7 100644 --- a/apps/rush/package.json +++ b/apps/rush/package.json @@ -48,6 +48,7 @@ "@rushstack/rush-amazon-s3-build-cache-plugin": "workspace:*", "@rushstack/rush-azure-storage-build-cache-plugin": "workspace:*", "@rushstack/rush-http-build-cache-plugin": "workspace:*", + "@rushstack/rush-serve-plugin": "workspace:*", "@types/heft-jest": "1.0.1", "@types/semver": "7.5.0" }, diff --git a/apps/rush/src/start-dev.ts b/apps/rush/src/start-dev.ts index 927fae7670f..bba3469421f 100644 --- a/apps/rush/src/start-dev.ts +++ b/apps/rush/src/start-dev.ts @@ -29,6 +29,7 @@ function includePlugin(pluginName: string, pluginPackageName?: string): void { includePlugin('rush-amazon-s3-build-cache-plugin'); includePlugin('rush-azure-storage-build-cache-plugin'); includePlugin('rush-http-build-cache-plugin'); +includePlugin('rush-serve-plugin'); // Including this here so that developers can reuse it without installing the plugin a second time includePlugin('rush-azure-interactive-auth-plugin', '@rushstack/rush-azure-storage-build-cache-plugin'); diff --git a/common/changes/@microsoft/rush/watch-rework_2025-09-26-23-50.json b/common/changes/@microsoft/rush/watch-rework_2025-09-26-23-50.json new file mode 100644 index 00000000000..ed3f1333a6a --- /dev/null +++ b/common/changes/@microsoft/rush/watch-rework_2025-09-26-23-50.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/rush", + "comment": "(PLUGIN BREAKING CHANGE) Overhaul watch-mode commands such that the graph is only created once at the start of command invocation, along with a stateful manager object. Plugins may now access the manager object and use it to orchestrate and tap into the build process.", + "type": "none" + } + ], + "packageName": "@microsoft/rush" +} \ No newline at end of file diff --git a/common/config/rush-plugins/rush-serve-plugin.json b/common/config/rush-plugins/rush-serve-plugin.json new file mode 100644 index 00000000000..8d24cc8cc26 --- /dev/null +++ b/common/config/rush-plugins/rush-serve-plugin.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/rush-serve-plugin-options.schema.json", + "phasedCommands": ["start"], + "portParameterLongName": "--port", + "buildStatusWebSocketPath": "/ws", + "logServePath": "/logs" +} diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json index 84354ed1fe6..a8941e8753b 100644 --- a/common/config/rush/command-line.json +++ b/common/config/rush/command-line.json @@ -245,7 +245,7 @@ // Used for very simple builds that don't support CLI arguments like `--production` or `--fix` "name": "_phase:lite-build", "dependencies": { - "upstream": ["_phase:lite-build", "_phase:build"] + "upstream": ["_phase:build"] }, "missingScriptBehavior": "silent", "allowWarningsOnSuccess": false @@ -253,8 +253,8 @@ { "name": "_phase:build", "dependencies": { - "self": ["_phase:lite-build"], - "upstream": ["_phase:build"] + // Don't need to declare the dependency on _phase:build because it is transitive via _phase:lite-build + "self": ["_phase:lite-build"] }, "missingScriptBehavior": "log", "allowWarningsOnSuccess": false @@ -262,7 +262,8 @@ { "name": "_phase:test", "dependencies": { - "self": ["_phase:lite-build", "_phase:build"] + // Dependency on _phase:lite-build is transitive via _phase:build + "self": ["_phase:build"] }, "missingScriptBehavior": "silent", "allowWarningsOnSuccess": false @@ -494,6 +495,14 @@ "associatedPhases": ["_phase:build", "_phase:test"], "associatedCommands": ["build", "rebuild", "test", "retest"] }, + { + "longName": "--port", + "parameterKind": "integer", + "argumentName": "PORT", + "description": "The port to use for the server", + "associatedPhases": [], + "associatedCommands": ["start"] + }, { "longName": "--update-snapshots", "parameterKind": "flag", diff --git a/common/config/rush/experiments.json b/common/config/rush/experiments.json index 7dcb8ecd683..3d0478fc99f 100644 --- a/common/config/rush/experiments.json +++ b/common/config/rush/experiments.json @@ -88,7 +88,7 @@ * of `_phase:` if they exist. The created child process will be provided with an IPC channel and expected to persist * across invocations. */ - // "useIPCScriptsInWatchMode": true, + "useIPCScriptsInWatchMode": true, /** * (UNDER DEVELOPMENT) The Rush alerts feature provides a way to send announcements to engineers diff --git a/common/config/rush/nonbrowser-approved-packages.json b/common/config/rush/nonbrowser-approved-packages.json index 4eb4bb89b8d..2b6a29b3df3 100644 --- a/common/config/rush/nonbrowser-approved-packages.json +++ b/common/config/rush/nonbrowser-approved-packages.json @@ -362,6 +362,10 @@ "name": "@rushstack/rush-sdk", "allowedCategories": [ "libraries", "tests", "vscode-extensions" ] }, + { + "name": "@rushstack/rush-serve-plugin", + "allowedCategories": [ "libraries" ] + }, { "name": "@rushstack/set-webpack-public-path-plugin", "allowedCategories": [ "libraries", "tests" ] diff --git a/common/config/subspaces/default/pnpm-lock.yaml b/common/config/subspaces/default/pnpm-lock.yaml index dd1378b412d..fadc2deaea9 100644 --- a/common/config/subspaces/default/pnpm-lock.yaml +++ b/common/config/subspaces/default/pnpm-lock.yaml @@ -417,6 +417,9 @@ importers: '@rushstack/rush-http-build-cache-plugin': specifier: workspace:* version: link:../../rush-plugins/rush-http-build-cache-plugin + '@rushstack/rush-serve-plugin': + specifier: workspace:* + version: link:../../rush-plugins/rush-serve-plugin '@types/heft-jest': specifier: 1.0.1 version: 1.0.1 @@ -781,7 +784,7 @@ importers: version: 6.4.22(@types/react@17.0.74)(react-dom@17.0.2(react@17.0.2))(react@17.0.2) '@storybook/cli': specifier: ~6.4.18 - version: 6.4.22(eslint@9.37.0)(jest@29.3.1(@types/node@20.17.19))(react-dom@17.0.2(react@17.0.2))(react@17.0.2)(typescript@5.8.2) + version: 6.4.22(eslint@9.37.0)(jest@29.3.1(@types/node@20.17.19)(babel-plugin-macros@3.1.0))(react-dom@17.0.2(react@17.0.2))(react@17.0.2)(typescript@5.8.2) '@storybook/components': specifier: ~6.4.18 version: 6.4.22(@types/react@17.0.74)(react-dom@17.0.2(react@17.0.2))(react@17.0.2) @@ -817,7 +820,7 @@ importers: version: 5.2.7(webpack@4.47.0) jest: specifier: ~29.3.1 - version: 29.3.1(@types/node@20.17.19) + version: 29.3.1(@types/node@20.17.19)(babel-plugin-macros@3.1.0) react: specifier: ~17.0.2 version: 17.0.2 @@ -1003,7 +1006,7 @@ importers: version: 5.2.7(webpack@5.105.2) jest: specifier: ~29.3.1 - version: 29.3.1(@types/node@20.17.19) + version: 29.3.1(@types/node@20.17.19)(babel-plugin-macros@3.1.0) react: specifier: ~19.2.3 version: 19.2.4 @@ -5165,6 +5168,9 @@ importers: '@rushstack/rush-sdk': specifier: workspace:* version: link:../../libraries/rush-sdk + '@rushstack/terminal': + specifier: workspace:* + version: link:../../libraries/terminal '@rushstack/ts-command-line': specifier: workspace:* version: link:../../libraries/ts-command-line @@ -5187,9 +5193,6 @@ importers: '@rushstack/heft': specifier: workspace:* version: link:../../apps/heft - '@rushstack/terminal': - specifier: workspace:* - version: link:../../libraries/terminal '@types/compression': specifier: ~1.7.2 version: 1.7.5(@types/express@4.17.21) @@ -22426,7 +22429,7 @@ snapshots: - supports-color - ts-node - '@jest/core@29.7.0': + '@jest/core@29.7.0(babel-plugin-macros@3.1.0)': dependencies: '@jest/console': 29.7.0 '@jest/reporters': 29.7.0 @@ -22440,7 +22443,7 @@ snapshots: exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@22.9.3) + jest-config: 29.7.0(@types/node@22.9.3)(babel-plugin-macros@3.1.0) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -25153,7 +25156,7 @@ snapshots: ts-dedent: 2.2.0 util-deprecate: 1.0.2 - '@storybook/cli@6.4.22(eslint@9.37.0)(jest@29.3.1(@types/node@20.17.19))(react-dom@17.0.2(react@17.0.2))(react@17.0.2)(typescript@5.8.2)': + '@storybook/cli@6.4.22(eslint@9.37.0)(jest@29.3.1(@types/node@20.17.19)(babel-plugin-macros@3.1.0))(react-dom@17.0.2(react@17.0.2))(react@17.0.2)(typescript@5.8.2)': dependencies: '@babel/core': 7.20.12 '@babel/preset-env': 7.28.6(@babel/core@7.20.12) @@ -25173,7 +25176,7 @@ snapshots: fs-extra: 9.1.0 get-port: 5.1.1 globby: 11.1.0 - jest: 29.3.1(@types/node@20.17.19) + jest: 29.3.1(@types/node@20.17.19)(babel-plugin-macros@3.1.0) jscodeshift: 0.13.1(@babel/preset-env@7.28.6(@babel/core@7.20.12)) json5: 2.2.3 leven: 3.1.0 @@ -28766,13 +28769,13 @@ snapshots: safe-buffer: 5.2.1 sha.js: 2.4.12 - create-jest@29.7.0(@types/node@20.17.19): + create-jest@29.7.0(@types/node@20.17.19)(babel-plugin-macros@3.1.0): dependencies: '@jest/types': 29.6.3 chalk: 4.1.2 exit: 0.1.2 graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@20.17.19) + jest-config: 29.7.0(@types/node@20.17.19)(babel-plugin-macros@3.1.0) jest-util: 29.7.0 prompts: 2.4.2 transitivePeerDependencies: @@ -32144,16 +32147,16 @@ snapshots: - babel-plugin-macros - supports-color - jest-cli@29.7.0(@types/node@20.17.19): + jest-cli@29.7.0(@types/node@20.17.19)(babel-plugin-macros@3.1.0): dependencies: - '@jest/core': 29.7.0 + '@jest/core': 29.7.0(babel-plugin-macros@3.1.0) '@jest/test-result': 29.7.0(@types/node@20.17.19) '@jest/types': 29.6.3 chalk: 4.1.2 - create-jest: 29.7.0(@types/node@20.17.19) + create-jest: 29.7.0(@types/node@20.17.19)(babel-plugin-macros@3.1.0) exit: 0.1.2 import-local: 3.2.0 - jest-config: 29.7.0(@types/node@20.17.19) + jest-config: 29.7.0(@types/node@20.17.19)(babel-plugin-macros@3.1.0) jest-util: 29.7.0 jest-validate: 29.7.0 yargs: 17.7.2 @@ -32223,7 +32226,7 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@20.17.19): + jest-config@29.7.0(@types/node@20.17.19)(babel-plugin-macros@3.1.0): dependencies: '@babel/core': 7.20.12 '@jest/test-sequencer': 29.7.0(@types/node@20.17.19) @@ -32253,7 +32256,7 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@22.9.3): + jest-config@29.7.0(@types/node@22.9.3)(babel-plugin-macros@3.1.0): dependencies: '@babel/core': 7.20.12 '@jest/test-sequencer': 29.7.0(@types/node@22.9.3) @@ -32641,12 +32644,12 @@ snapshots: merge-stream: 2.0.0 supports-color: 8.1.1 - jest@29.3.1(@types/node@20.17.19): + jest@29.3.1(@types/node@20.17.19)(babel-plugin-macros@3.1.0): dependencies: '@jest/core': 29.5.0(babel-plugin-macros@3.1.0) '@jest/types': 29.5.0 import-local: 3.2.0 - jest-cli: 29.7.0(@types/node@20.17.19) + jest-cli: 29.7.0(@types/node@20.17.19)(babel-plugin-macros@3.1.0) transitivePeerDependencies: - '@types/node' - babel-plugin-macros diff --git a/common/config/subspaces/default/repo-state.json b/common/config/subspaces/default/repo-state.json index 19c5e725c7c..5cf4edb86c1 100644 --- a/common/config/subspaces/default/repo-state.json +++ b/common/config/subspaces/default/repo-state.json @@ -1,5 +1,5 @@ // DO NOT MODIFY THIS FILE MANUALLY BUT DO COMMIT IT. It is generated and used by Rush. { - "pnpmShrinkwrapHash": "8cea08431b1a848463c751666dcd35e9533ea73d", + "pnpmShrinkwrapHash": "b595f703fea0b5d0ca523cf2bc67e30981b3d0a3", "preferredVersionsHash": "029c99bd6e65c5e1f25e2848340509811ff9753c" } diff --git a/common/reviews/api/rush-lib.api.md b/common/reviews/api/rush-lib.api.md index 14063d1543e..9fc083cff87 100644 --- a/common/reviews/api/rush-lib.api.md +++ b/common/reviews/api/rush-lib.api.md @@ -32,6 +32,7 @@ import type { StdioSummarizer } from '@rushstack/terminal'; import { SyncHook } from 'tapable'; import { SyncWaterfallHook } from 'tapable'; import { Terminal } from '@rushstack/terminal'; +import type { TerminalWritable } from '@rushstack/terminal'; // @public export class ApprovedPackagesConfiguration { @@ -329,6 +330,14 @@ export type GetCacheEntryIdFunction = (options: IGenerateCacheEntryIdOptions) => // @beta export type GetInputsSnapshotAsyncFn = () => Promise; +// @alpha (undocumented) +export interface IBaseOperationExecutionResult { + getStateHash(): string; + getStateHashComponents(): IOperationStateHashComponents; + readonly metadataFolderPath: string; + readonly operation: Operation; +} + // @internal (undocumented) export interface _IBuiltInPluginConfiguration extends _IRushPluginConfigurationBase { // (undocumented) @@ -389,6 +398,11 @@ export interface ICobuildLockProvider { setCompletedStateAsync(context: Readonly, state: ICobuildCompletedState): Promise; } +// @alpha +export interface IConfigurableOperation extends IBaseOperationExecutionResult { + enabled: boolean; +} + // @public export interface IConfigurationEnvironment { [environmentVariableName: string]: IConfigurationEnvironmentVariable; @@ -406,17 +420,14 @@ export interface ICreateOperationsContext { readonly changedProjectsOnly: boolean; readonly cobuildConfiguration: CobuildConfiguration | undefined; readonly customParameters: ReadonlyMap; + readonly generateFullGraph?: boolean; readonly includePhaseDeps: boolean; - readonly invalidateOperation?: ((operation: Operation, reason: string) => void) | undefined; readonly isIncrementalBuildAllowed: boolean; - readonly isInitial: boolean; readonly isWatch: boolean; - readonly parallelism: number; - readonly phaseOriginal: ReadonlySet; + readonly parallelism: Parallelism; readonly phaseSelection: ReadonlySet; readonly projectConfigurations: ReadonlyMap; readonly projectSelection: ReadonlySet; - readonly projectsInUnknownState: ReadonlySet; readonly rushConfiguration: RushConfiguration; } @@ -450,12 +461,6 @@ export interface IEnvironmentConfigurationInitializeOptions { doNotNormalizePaths?: boolean; } -// @alpha -export interface IExecuteOperationsContext extends ICreateOperationsContext { - readonly abortController: AbortController; - readonly inputsSnapshot?: IInputsSnapshot; -} - // @alpha export interface IExecutionResult { readonly operationResults: ReadonlyMap; @@ -591,14 +596,11 @@ export interface _IOperationBuildCacheOptions { } // @alpha -export interface IOperationExecutionResult { +export interface IOperationExecutionResult extends IBaseOperationExecutionResult, IOperationLastState { + readonly enabled: boolean; readonly error: Error | undefined; - getStateHash(): string; - getStateHashComponents(): ReadonlyArray; readonly logFilePaths: ILogFilePaths | undefined; - readonly metadataFolderPath: string | undefined; readonly nonCachedDurationMs: number | undefined; - readonly operation: Operation; readonly problemCollector: IProblemCollector; readonly silent: boolean; readonly status: OperationStatus; @@ -606,6 +608,48 @@ export interface IOperationExecutionResult { readonly stopwatch: IStopwatchResult; } +// @alpha +export interface IOperationGraph { + readonly abortController: AbortController; + abortCurrentIterationAsync(): Promise; + addTerminalDestination(destination: TerminalWritable): void; + allowOversubscription: boolean; + closeRunnersAsync(operations?: Iterable): Promise; + debugMode: boolean; + executeScheduledIterationAsync(): Promise; + readonly hasScheduledIteration: boolean; + readonly hooks: OperationGraphHooks; + invalidateOperations(operations?: Iterable, reason?: string): void; + readonly operations: ReadonlySet; + get parallelism(): number; + set parallelism(value: Parallelism); + pauseNextIteration: boolean; + quietMode: boolean; + removeTerminalDestination(destination: TerminalWritable, close?: boolean): boolean; + readonly resultByOperation: ReadonlyMap; + scheduleIterationAsync(options: IOperationGraphIterationOptions): Promise; + setEnabledStates(operations: Iterable, targetState: Operation['enabled'], mode: 'safe' | 'unsafe'): boolean; + readonly status: OperationStatus; + readonly terminalDestinations: ReadonlySet; +} + +// @alpha +export interface IOperationGraphContext extends ICreateOperationsContext { + readonly initialSnapshot?: IInputsSnapshot; +} + +// @alpha +export interface IOperationGraphIterationOptions { + // (undocumented) + inputsSnapshot?: IInputsSnapshot; + startTime?: number; +} + +// @beta +export interface IOperationLastState { + readonly status: OperationStatus; +} + // @internal (undocumented) export interface _IOperationMetadata { // (undocumented) @@ -630,6 +674,7 @@ export interface _IOperationMetadataManagerOptions { // @alpha export interface IOperationOptions { + enabled?: OperationEnabledState; logFilenameIdentifier: string; phase: IPhase; project: RushConfigurationProject; @@ -640,8 +685,10 @@ export interface IOperationOptions { // @beta export interface IOperationRunner { cacheable: boolean; - executeAsync(context: IOperationRunnerContext): Promise; + closeAsync?(): Promise; + executeAsync(context: IOperationRunnerContext, lastState?: IOperationLastState): Promise; getConfigHash(): string; + readonly isActive?: boolean; readonly isNoOp?: boolean; readonly name: string; reportTiming: boolean; @@ -655,6 +702,7 @@ export interface IOperationRunnerContext { debugMode: boolean; environment: IEnvironment | undefined; error?: Error; + getInvalidateCallback(): (reason: string) => void; // @internal _operationMetadataManager: _OperationMetadataManager; quietMode: boolean; @@ -689,6 +737,13 @@ export interface _IOperationStateFileOptions { projectFolder: string; } +// @alpha +export interface IOperationStateHashComponents { + readonly config: string; + readonly dependencies: readonly string[]; + readonly local: string; +} + // @internal (undocumented) export interface _IOperationStateJson { // (undocumented) @@ -704,6 +759,12 @@ export interface IPackageManagerOptionsJsonBase { environmentVariables?: IConfigurationEnvironment; } +// @beta +export interface IParallelismScalar { + // (undocumented) + readonly scalar: number; +} + // @alpha export interface IPhase { allowWarningsOnSuccess: boolean; @@ -730,6 +791,11 @@ export interface IPhasedCommand extends IRushCommand { readonly sessionAbortController: AbortController; } +// @alpha +export interface IPhasedCommandPlugin { + apply(hooks: PhasedCommandHooks): void; +} + // @public export interface IPnpmLockfilePolicies { disallowInsecureSha1?: { @@ -982,13 +1048,13 @@ export class Operation { readonly consumers: ReadonlySet; deleteDependency(dependency: Operation): void; readonly dependencies: ReadonlySet; - enabled: boolean; + enabled: OperationEnabledState; get isNoOp(): boolean; logFilenameIdentifier: string; get name(): string; runner: IOperationRunner | undefined; settings: IOperationSettings | undefined; - weight: number; + weight: Parallelism; } // @internal (undocumented) @@ -996,7 +1062,7 @@ export class _OperationBuildCache { // (undocumented) get cacheId(): string | undefined; // (undocumented) - static forOperation(executionResult: IOperationExecutionResult, options: _IOperationBuildCacheOptions): _OperationBuildCache; + static forOperation(executionResult: IBaseOperationExecutionResult, options: _IOperationBuildCacheOptions): _OperationBuildCache; // (undocumented) static getOperationBuildCache(options: _IProjectBuildCacheOptions): _OperationBuildCache; // (undocumented) @@ -1005,6 +1071,44 @@ export class _OperationBuildCache { trySetCacheEntryAsync(terminal: ITerminal, specifiedCacheId?: string): Promise; } +// @alpha +export type OperationEnabledState = boolean | 'ignore-dependency-changes'; + +// @alpha +export class OperationGraphHooks { + readonly afterExecuteIterationAsync: AsyncSeriesWaterfallHook<[ + OperationStatus, + ReadonlyMap, + IOperationGraphIterationOptions + ]>; + readonly afterExecuteOperationAsync: AsyncSeriesHook<[ + IOperationRunnerContext & IOperationExecutionResult + ]>; + readonly beforeExecuteIterationAsync: AsyncSeriesBailHook<[ + ReadonlyMap, + IOperationGraphIterationOptions + ], OperationStatus | undefined | void>; + readonly beforeExecuteOperationAsync: AsyncSeriesBailHook<[ + IOperationRunnerContext & IOperationExecutionResult + ], OperationStatus | undefined>; + readonly beforeLog: SyncHook; + readonly configureIteration: SyncHook<[ + ReadonlyMap, + ReadonlyMap, + IOperationGraphIterationOptions + ]>; + readonly createEnvironmentForOperation: SyncWaterfallHook<[ + IEnvironment, + IOperationRunnerContext & IOperationExecutionResult + ]>; + readonly onEnableStatesChanged: SyncHook<[ReadonlySet]>; + readonly onExecutionStatesUpdated: SyncHook<[ReadonlySet]>; + readonly onGraphStateChanged: SyncHook<[IOperationGraph]>; + readonly onIdle: SyncHook; + readonly onInvalidateOperations: SyncHook<[Iterable, string | undefined]>; + readonly onIterationScheduled: SyncHook<[ReadonlyMap]>; +} + // @internal export class _OperationMetadataManager { constructor(options: _IOperationMetadataManagerOptions); @@ -1132,28 +1236,16 @@ export abstract class PackageManagerOptionsConfigurationBase implements IPackage readonly environmentVariables?: IConfigurationEnvironment; } +// @beta +export type Parallelism = number | IParallelismScalar; + // @alpha export class PhasedCommandHooks { - readonly afterExecuteOperation: AsyncSeriesHook<[ - IOperationRunnerContext & IOperationExecutionResult - ]>; - readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, IExecuteOperationsContext]>; - readonly beforeExecuteOperation: AsyncSeriesBailHook<[ - IOperationRunnerContext & IOperationExecutionResult - ], OperationStatus | undefined>; - readonly beforeExecuteOperations: AsyncSeriesHook<[ - Map, - IExecuteOperationsContext - ]>; - readonly beforeLog: SyncHook; - readonly createEnvironmentForOperation: SyncWaterfallHook<[ - IEnvironment, - IOperationRunnerContext & IOperationExecutionResult + readonly createOperationsAsync: AsyncSeriesWaterfallHook<[ + Set, + ICreateOperationsContext ]>; - readonly createOperations: AsyncSeriesWaterfallHook<[Set, ICreateOperationsContext]>; - readonly onOperationStatusChanged: SyncHook<[IOperationExecutionResult]>; - readonly shutdownAsync: AsyncParallelHook; - readonly waitingForChanges: SyncHook; + readonly onGraphCreatedAsync: AsyncSeriesHook<[IOperationGraph, IOperationGraphContext]>; } // @public diff --git a/docs/rush/phased-commands.md b/docs/rush/phased-commands.md new file mode 100644 index 00000000000..9a4908d53fd --- /dev/null +++ b/docs/rush/phased-commands.md @@ -0,0 +1,436 @@ +# Phased Command Execution and Plugin Architecture + +This document describes the architecture for Rush's phased command system, including how commands execute, how the operation graph is managed, and how plugins can hook into the process. + +## Overview + +A **phased command** (e.g. `rush build`, `rush start`) runs a set of **operations** across projects, potentially in parallel, potentially in **watch mode** where the command runs indefinitely and re-executes operations when source files change. + +The key classes involved are: + +- `PhasedScriptAction` — parses CLI args, orchestrates the command lifecycle, owns `PhasedCommandHooks` +- `OperationGraph` — manages the stateful execution of operations across the entire session +- `ProjectWatcher` — drives watch mode by observing file system changes and scheduling new iterations +- `PhasedCommandHooks` — the plugin API surface exposed to Rush plugins via `runAnyPhasedCommand` / `runPhasedCommand` + +--- + +## Command Lifecycle + +### 1. Plugin registration + +Before any work begins, `PhasedScriptAction.runAsync()` applies built-in plugins and fires session hooks: + +```text +hooks.runAnyPhasedCommand → hooks.runPhasedCommand[actionName] +``` + +Built-in plugins applied (in order): + +1. `PhasedOperationPlugin` — generates the default operation graph from phases and project selection +2. `ShardedPhasedOperationPlugin` — splices in sharded phases +3. `ShellOperationRunnerPlugin` — assigns `ShellOperationRunner` to operations with scripts +4. `ValidateOperationsPlugin` — validates `rush-project.json` entries +5. Conditional plugins: `ConsoleTimelinePlugin`, `NodeDiagnosticDirPlugin`, `OperationResultSummarizerPlugin` +6. Cache plugins (one of): `CacheableOperationPlugin`, `LegacySkipPlugin`, or none (full rebuild) +7. `IPCOperationRunnerPlugin` — in watch mode, enables long-running processes via IPC + +### 2. Graph creation (`createOperationsAsync`) + +`PhasedCommandHooks.createOperationsAsync` is fired with an empty `Set` and an `ICreateOperationsContext`. Each tap in the waterfall may add, remove, or mutate operations. **This hook is invoked exactly once per session**, regardless of how many watch iterations occur. The resulting `Operation` objects are reused for the entire lifetime of the session. Operations cannot be added to or removed from the graph after this hook completes. + +**`ICreateOperationsContext` fields:** + +| Field | Description | +| --- | --- | +| `buildCacheConfiguration` | Build cache config, if enabled | +| `changedProjectsOnly` | Whether `--changed-projects-only` was passed | +| `cobuildConfiguration` | Cobuild config, if enabled | +| `customParameters` | Map of longName → CLI parameter | +| `includePhaseDeps` | Whether phase dependencies are auto-included | +| `isIncrementalBuildAllowed` | False for `rush rebuild` | +| `isWatch` | True if running in watch mode | +| `parallelism` | Configured max parallelism | +| `phaseSelection` | Set of phases to run | +| `projectConfigurations` | Loaded `rush-project.json` data | +| `projectSelection` | Set of selected projects | +| `generateFullGraph` | True when `includeAllProjectsInWatchGraph` is set and in watch mode | +| `rushConfiguration` | The Rush configuration | + +### 3. OperationGraph construction + +After `createOperationsAsync`, Rush constructs an `OperationGraph` from the resulting operations and fires: + +```text +PhasedCommandHooks.onGraphCreatedAsync(operationGraph: IOperationGraph, context: IOperationGraphContext) +``` + +`IOperationGraphContext` extends `ICreateOperationsContext` with: + +- `initialSnapshot?: IInputsSnapshot` — the current file system state (used to seed incremental build detection) + +Plugins that tap `onGraphCreatedAsync` should register their hooks on `operationGraph.hooks` (`OperationGraphHooks`) for all per-iteration and per-operation behavior. + +### 4. Execution + +**Non-watch mode:** `graph.executeAsync(iterationOptions)` returns a `Promise` that resolves when the single iteration has finished. `IExecutionResult` contains: + +- `status: OperationStatus` — the overall outcome (`Success`, `Failure`, `NoOp`, etc.) +- `operationResults: ReadonlyMap` — per-operation results for the iteration + +**Watch mode:** `graph.executeAsync(iterationOptions)` returns a `Promise` for the **initial** iteration only. After that promise resolves, a `ProjectWatcher` observes file system changes. When changes are detected (after a debounce), `ProjectWatcher` calls `graph.scheduleIterationAsync(...)`, which queues a new iteration. Subsequent iterations are driven internally and their results are available via `graph.resultByOperation`. After each iteration completes with no queued work, `graph.hooks.onIdle` fires and the graph enters an idle state until the next change. + +--- + +## OperationGraphHooks + +All hooks in `OperationGraphHooks` are accessible via `operationGraph.hooks`. These are registered during `onGraphCreatedAsync`. + +### Per-iteration hooks + +#### `configureIteration` (Sync) + +```ts +SyncHook<[ + ReadonlyMap, // initialRecords — mutable enabled state + ReadonlyMap, // lastExecutedRecords — results from prior run + IOperationGraphIterationOptions +]> +``` + +Called synchronously before an iteration is queued. Use this to enable/disable operations based on which projects changed. **Must be synchronous** — the graph may be mid-execution when this fires and the `lastExecutedRecords` map must remain stable. + +When `lastExecutedRecords` is empty, this is the first iteration of the session. An operation has no entry in `lastExecutedRecords` if it has never reached a completed terminal state (`Success`, `SuccessWithWarning`, `Failure`, `FromCache`, or `NoOp`) — for example if it was `Aborted`, `Blocked`, or `Skipped` in all prior iterations. + +#### `onIterationScheduled` (Sync) + +```ts +SyncHook<[ReadonlyMap]> +``` + +Fires after an iteration is scheduled but before any operations execute. Useful for snapshotting planned work or pre-computing auxiliary data (e.g. dashboard rendering). + +#### `beforeExecuteIterationAsync` (AsyncSeriesBail) + +```ts +AsyncSeriesBailHook< + [ReadonlyMap, IOperationGraphIterationOptions], + OperationStatus | undefined | void +> +``` + +Fires at the start of executing a scheduled iteration. If any tap returns an `OperationStatus`, the remaining taps are skipped and the iteration ends immediately with that status; all non-started operations are marked `Aborted`. + +#### `afterExecuteIterationAsync` (AsyncSeriesWaterfall) + +```ts +AsyncSeriesWaterfallHook<[ + OperationStatus, + ReadonlyMap, + IOperationGraphIterationOptions +]> +``` + +Fires after all operations in an iteration have reached a final state. Taps may modify the `OperationStatus` that is returned. + +#### `onIdle` (Sync) + +Fires when the graph is idle and watching for file changes. Only relevant in watch mode. + +### Per-operation hooks + +#### `beforeExecuteOperationAsync` (AsyncSeriesBail) + +```ts +AsyncSeriesBailHook< + [IOperationRunnerContext & IOperationExecutionResult], + OperationStatus | undefined +> +``` + +Fires before executing a single operation. If a tap returns a status, the runner is skipped and the operation is assigned that status (used by cache plugins to short-circuit with `FromCache`). + +#### `afterExecuteOperationAsync` (AsyncSeries) + +```ts +AsyncSeriesHook<[IOperationRunnerContext & IOperationExecutionResult]> +``` + +Fires after a single operation completes. + +#### `createEnvironmentForOperation` (SyncWaterfall) + +```ts +SyncWaterfallHook<[IEnvironment, IOperationRunnerContext & IOperationExecutionResult]> +``` + +Called to construct the environment variables passed to the operation's shell runner. Taps can add, remove, or override environment variables. + +### State change hooks + +#### `onExecutionStatesUpdated` (Sync) + +```ts +SyncHook<[ReadonlySet]> +``` + +Batched hook invoked when one or more operation statuses change within the same microtask. Rather than firing once per individual status change, all changes that occur within a single microtask are collected into one set and delivered together. This reduces redundant renders or notifications when many operations update simultaneously (e.g. when a batch of operations are marked `Blocked` after an upstream failure). + +#### `onEnableStatesChanged` (Sync) + +```ts +SyncHook<[ReadonlySet]> +``` + +Fires when `IOperationGraph.setEnabledStates()` changes the `enabled` flag on any operations. + +#### `onGraphStateChanged` (Sync) + +```ts +SyncHook<[IOperationGraph]> +``` + +Fires when any observable property of the graph changes (parallelism, quiet/debug mode, `pauseNextIteration`, status, scheduled iteration availability). Used to drive reactive UIs. + +#### `onInvalidateOperations` (Sync) + +```ts +SyncHook<[Iterable, string | undefined]> +``` + +Fires when `IOperationGraph.invalidateOperations()` marks operations as `Ready` for re-execution. + +--- + +## IOperationGraph API + +`IOperationGraph` (implemented by `OperationGraph`) is the main handle for plugins to interact with the execution session at runtime. + +### Configuration properties + +| Property | Description | +| --- | --- | +| `parallelism` | Max concurrent operations (writable) | +| `debugMode` | Verbose debug output (writable) | +| `quietMode` | Suppress per-operation output except errors (writable) | +| `pauseNextIteration` | When true, scheduled iterations will not auto-execute (writable) | + +### Read-only state + +| Property | Description | +| --- | --- | +| `operations` | All operations in the graph (session-long set) | +| `resultByOperation` | Per-operation result records, updated live as each operation executes | +| `status` | Overall execution status (`Ready`, `Executing`, `Success`, `Failure`, etc.) | +| `hasScheduledIteration` | True if an iteration is queued but not yet running | +| `abortController` | Session-level `AbortController`; abort this to terminate watch mode | + +### Methods + +| Method | Description | +| --- | --- | +| `scheduleIterationAsync(options)` | Queue a new iteration; returns `true` if scheduled, `false` if nothing to do | +| `executeScheduledIterationAsync()` | Execute the currently queued iteration | +| `executeAsync(options)` | Convenience: schedule + execute in one call; returns `Promise` with `status` and `operationResults` | +| `abortCurrentIterationAsync()` | Cancel the in-flight iteration | +| `invalidateOperations(operations?, reason?)` | Mark operations as needing re-execution | +| `setEnabledStates(operations, targetState, mode)` | Enable or disable operations (`'safe'` mode respects dependencies) | +| `closeRunnersAsync(operations?)` | Dispose long-running IPC runners | +| `addTerminalDestination(dest)` | Attach an additional `TerminalWritable` for output | +| `removeTerminalDestination(dest, close?)` | Detach a terminal destination | + +--- + +## Watch Mode — `includeAllProjectsInWatchGraph` + +When the `watchOptions.includeAllProjectsInWatchGraph` flag is set to `true` in `command-line.json`, Rush builds the operation graph with **all projects** in `rush.json`, not just the CLI-selected subset. The `generateFullGraph` property on `ICreateOperationsContext` reflects this. + +This enables plugins (such as `rush-serve-plugin`) to dynamically enable/disable individual operations in the graph during the watch session by calling `IOperationGraph.setEnabledStates()`. For example, an HTTP server or WebSocket endpoint can receive a message saying "enable project X" and call `setEnabledStates` to bring it into the build graph without needing to restart the watch session. + +--- + +## IOperationRunner + +Each `Operation` carries a `runner: IOperationRunner` that performs the actual work. In watch mode the same runner instance is called once per iteration for the lifetime of the session, so runners must be written to handle multiple invocations. + +### `executeAsync(context, lastState?)` + +```ts +executeAsync(context: IOperationRunnerContext, lastState?: {}): Promise +``` + +`lastState` is the `IOperationExecutionResult` from the most recent iteration in which this operation **reached a completed terminal state** (`Success`, `SuccessWithWarning`, `Failure`, `FromCache`, or `NoOp`). It is `undefined` when: + +- This is the first time the runner has ever been called, or +- Every prior iteration either did not reach this operation (aborted before execution began) or left it in a non-completing state (`Skipped`, `Blocked`, `Aborted`). + +Runners use `lastState` to choose between a full initial build and an incremental build: + +```ts +async executeAsync(context: IOperationRunnerContext, lastState?: {}): Promise { + if (lastState === undefined) { + // No completed prior result — run full build + } else { + // Prior result exists; check lastState.status if the outcome matters + // e.g. re-run full build if the previous run failed + } +} +``` + +`ShellOperationRunner` uses this to select between the `initialCommand` and `incrementalCommand` scripts defined in `rush-project.json`. + +### `isActive` + +```ts +readonly isActive?: boolean; +``` + +Optional. Set to `true` while the runner owns a live background resource (e.g. a running dev server or file watcher). Tooling such as the live dashboard uses this to represent an operation as "in progress" even when it is not currently executing an iteration. The runner is responsible for updating this property as the resource starts and stops. + +### `closeAsync` + +```ts +closeAsync?(): Promise; +``` + +Optional. Called by `IOperationGraph.closeRunnersAsync()` when the session ends (triggered by the `abortController` abort signal). Must be **idempotent** — it may be called before any `executeAsync` call has occurred, or after the runner has already been closed. Failing to implement it defensively can cause errors during watch-mode teardown. + +### Long-lived runner example + +```ts +class MyWatchRunner implements IOperationRunner { + readonly name: string; + cacheable = false; + reportTiming = true; + silent = false; + warningsAreAllowed = false; + + private _server: MyServer | undefined; + + get isActive(): boolean { + return this._server !== undefined; + } + + async executeAsync(context: IOperationRunnerContext, lastState?: {}): Promise { + if (!this._server) { + this._server = await MyServer.startAsync(); + } else { + await this._server.reloadAsync(); + } + return OperationStatus.Success; + } + + getConfigHash(): string { return ''; } + + async closeAsync(): Promise { + await this._server?.stopAsync(); + this._server = undefined; + } +} +``` + +--- + +## Plugin patterns + +### Session context in graph hooks + +Session-scoped data from `IOperationGraphContext` (build cache config, project selection, phase selection, etc.) is available in the `onGraphCreatedAsync` callback and can be captured by closure for use in all graph hooks registered within it: + +```ts +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph, context) => { + const { buildCacheConfiguration, projectSelection } = context; + + graph.hooks.beforeExecuteIterationAsync.tapPromise(PLUGIN_NAME, async (records, iterationOptions) => { + // buildCacheConfiguration and projectSelection available via closure + }); +}); +``` + +### Self-invalidation from runners + +Long-lived runners (e.g. file watchers, IPC processes) that detect stale outputs between iterations can request re-execution via `context.getInvalidateCallback()`. This returns a `(reason: string) => void` callback that delegates to `IOperationGraph.invalidateOperations()` under the hood, marking the operation as `Ready` and scheduling a new iteration. The returned callback captures only the minimal state needed, so callers should store it rather than retaining the full context. + +Because `context` is always available (not just after the first completed run), runners can obtain the callback on their very first execution — there is no need to wait for a previous result. + +```ts +class MyWatchRunner implements IOperationRunner { + async executeAsync(context: IOperationRunnerContext, lastState?: {}): Promise { + const invalidate = context.getInvalidateCallback(); + // ... do work, set up watchers that call invalidate('files changed') + return OperationStatus.Success; + } +} +``` + +### Session teardown + +To run cleanup when the watch session ends, listen to the abort signal on the graph: + +```ts +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.abortController.signal.addEventListener('abort', () => { + void myResource.dispose(); + }, { once: true }); +}); +``` + +Note that there is no built-in mechanism to await parallel teardown tasks before the process exits. `graph.closeRunnersAsync()` awaits all runner `closeAsync` calls, but other async cleanup must be coordinated manually. + +--- + +## Plugin example + +```ts +import type { IPhasedCommandPlugin, PhasedCommandHooks } from '@microsoft/rush-lib'; + +export class MyPlugin implements IPhasedCommandPlugin { + apply(hooks: PhasedCommandHooks): void { + // Step 1: Add operations to the graph + hooks.createOperationsAsync.tapPromise('MyPlugin', async (operations, context) => { + // Inspect context, optionally add new Operations + return operations; + }); + + // Step 2: Tap into the graph after it is created + hooks.onGraphCreatedAsync.tapPromise('MyPlugin', async (graph, context) => { + // Configure per-iteration behavior + graph.hooks.configureIteration.tap('MyPlugin', (initialRecords, lastResults, iterationContext) => { + // Enable/disable operations based on what changed + }); + + graph.hooks.beforeExecuteIterationAsync.tapPromise('MyPlugin', async (results, iterationContext) => { + // Optionally short-circuit the iteration + }); + + graph.hooks.afterExecuteIterationAsync.tapPromise('MyPlugin', async (status, results, context) => { + // Post-iteration reporting + return status; + }); + + graph.hooks.onIdle.tap('MyPlugin', () => { + // Display idle status + }); + + graph.hooks.onGraphStateChanged.tap('MyPlugin', (operationGraph) => { + // React to property changes (good for live dashboards) + }); + }); + } +} +``` + +--- + +## Key source files + +| File | Description | +| --- | --- | +| `libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts` | All hook and interface definitions for phased commands | +| `libraries/rush-lib/src/logic/operations/OperationGraph.ts` | `OperationGraph` implementation (`IOperationGraph`) | +| `libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts` | Command entry point; orchestrates the full lifecycle | +| `libraries/rush-lib/src/logic/ProjectWatcher.ts` | File system watcher; drives watch mode iterations | +| `libraries/rush-lib/src/logic/operations/Operation.ts` | `Operation` node in the graph | +| `libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts` | Per-iteration execution state for an operation | +| `libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts` | Result interfaces | +| `libraries/rush-lib/src/logic/operations/IOperationRunner.ts` | `IOperationRunner` interface — `executeAsync`, `lastState`, `isActive`, `closeAsync` | diff --git a/docs/rush/plugin-migration-guide.md b/docs/rush/plugin-migration-guide.md new file mode 100644 index 00000000000..5ea09366bb3 --- /dev/null +++ b/docs/rush/plugin-migration-guide.md @@ -0,0 +1,419 @@ +# Rush Plugin Migration Guide: Phased Command Hooks + +This guide covers the breaking changes to the Rush plugin API for phased commands introduced in the watch-mode overhaul. The execution engine is now stateful across an entire Rush watch session, and the hook surface has been reorganized accordingly. + +## Summary of changes + +| Old (removed) | New | +| --- | --- | +| `PhasedCommandHooks.createOperations` | `PhasedCommandHooks.createOperationsAsync` | +| `PhasedCommandHooks.beforeExecuteOperations` | `operationGraph.hooks.beforeExecuteIterationAsync` | +| `PhasedCommandHooks.afterExecuteOperations` | `operationGraph.hooks.afterExecuteIterationAsync` | +| `PhasedCommandHooks.onOperationStatusChanged` | `operationGraph.hooks.onExecutionStatesUpdated` | +| `PhasedCommandHooks.beforeExecuteOperation` | `operationGraph.hooks.beforeExecuteOperationAsync` | +| `PhasedCommandHooks.afterExecuteOperation` | `operationGraph.hooks.afterExecuteOperationAsync` | +| `PhasedCommandHooks.createEnvironmentForOperation` | `operationGraph.hooks.createEnvironmentForOperation` | +| `PhasedCommandHooks.waitingForChanges` | `operationGraph.hooks.onIdle` | +| `PhasedCommandHooks.shutdownAsync` | `IOperationGraph.abortController` signal + `closeRunnersAsync()` | +| `PhasedCommandHooks.beforeLog` | `operationGraph.hooks.beforeLog` | +| `IExecuteOperationsContext` | `IOperationGraphIterationOptions` (iteration scope) + `IOperationGraphContext` (session scope) | +| `WeightedOperationPlugin` | Removed — weight assignment is now part of `PhasedOperationPlugin` | + +--- + +## Core migration pattern + +Previously, all hooks were tapped directly on the `PhasedCommandHooks` object passed to `apply()`. All per-iteration and per-operation hooks have moved to a new `OperationGraphHooks` class that lives on the `IOperationGraph` object, which is created once per session and persists across watch iterations. + +The new entry point for these hooks is `PhasedCommandHooks.onGraphCreatedAsync`. Register all graph-level hook taps inside this callback. + +### Before + +```typescript +export class MyPlugin implements IPhasedCommandPlugin { + apply(hooks: PhasedCommandHooks): void { + hooks.createOperations.tapPromise(PLUGIN_NAME, async (operations, context) => { + // mutate operations + return operations; + }); + + hooks.beforeExecuteOperations.tapPromise(PLUGIN_NAME, async (records, context) => { + // runs before each iteration + }); + + hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async (result, context) => { + // runs after each iteration + }); + + hooks.beforeExecuteOperation.tapPromise(PLUGIN_NAME, async (record) => { + // runs before a single operation + return undefined; + }); + + hooks.afterExecuteOperation.tapPromise(PLUGIN_NAME, async (record) => { + // runs after a single operation + }); + + hooks.waitingForChanges.tap(PLUGIN_NAME, () => { + // watch mode idle + }); + } +} +``` + +### After + +```typescript +export class MyPlugin implements IPhasedCommandPlugin { + apply(hooks: PhasedCommandHooks): void { + hooks.createOperationsAsync.tapPromise(PLUGIN_NAME, async (operations, context) => { + // mutate operations + return operations; + }); + + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph, context) => { + graph.hooks.beforeExecuteIterationAsync.tapPromise(PLUGIN_NAME, async (records, iterationContext) => { + // runs before each iteration + }); + + graph.hooks.afterExecuteIterationAsync.tapPromise(PLUGIN_NAME, async (status, records, iterationContext) => { + // runs after each iteration + return status; + }); + + graph.hooks.beforeExecuteOperationAsync.tapPromise(PLUGIN_NAME, async (record) => { + // runs before a single operation + return undefined; + }); + + graph.hooks.afterExecuteOperationAsync.tapPromise(PLUGIN_NAME, async (record) => { + // runs after a single operation + }); + + graph.hooks.onIdle.tap(PLUGIN_NAME, () => { + // watch mode idle + }); + }); + } +} +``` + +--- + +## Hook-by-hook migration + +### `createOperations` → `createOperationsAsync` + +The hook is now async-only and has been renamed to reflect this. + +**Critically, `createOperationsAsync` is now invoked exactly once per session**, regardless of how many watch iterations occur. Previously, each watch iteration called `createOperations` anew; now the same set of `Operation` objects is reused for the entire session. Plugins must not assume the hook will fire again after the initial call. Any per-iteration logic that was previously placed in `createOperations` (e.g. inspecting `isInitial` or `projectsInUnknownState`) must move to `operationGraph.hooks.configureIteration`. + +```typescript +// Before +hooks.createOperations.tap(PLUGIN_NAME, (operations, context) => { + return operations; +}); + +// After +hooks.createOperationsAsync.tap(PLUGIN_NAME, (operations, context) => { + return operations; +}); +// or async: +hooks.createOperationsAsync.tapPromise(PLUGIN_NAME, async (operations, context) => { + return operations; +}); +``` + +### `beforeExecuteOperations` → `operationGraph.hooks.beforeExecuteIterationAsync` + +The context parameter has changed type. `IExecuteOperationsContext` is replaced by `IOperationGraphIterationOptions`, which only carries the iteration-scoped data (`inputsSnapshot` and `startTime`). Session-scoped data (build cache config, project selection, etc.) is available via closure from the `onGraphCreatedAsync` callback parameter. + +The return signature of `beforeExecuteIterationAsync` is a bail hook: returning an `OperationStatus` short-circuits the iteration immediately. + +```typescript +// Before +hooks.beforeExecuteOperations.tapPromise(PLUGIN_NAME, async (records, context) => { + const { inputsSnapshot } = context; + // context also had: isInitial, projectsInUnknownState, etc. +}); + +// After +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph, sessionContext) => { + graph.hooks.beforeExecuteIterationAsync.tapPromise( + PLUGIN_NAME, + async (records, iterationOptions) => { + const { inputsSnapshot } = iterationOptions; + // sessionContext has: buildCacheConfiguration, projectSelection, etc. + // Return an OperationStatus to abort the iteration early, or return undefined to continue. + return undefined; + } + ); +}); +``` + +### `afterExecuteOperations` → `operationGraph.hooks.afterExecuteIterationAsync` + +The result parameter changed from `IExecutionResult` (an object with `status` and `operationResults`) to two separate parameters. The hook is now a waterfall on the status value. + +```typescript +// Before +hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async (result, context) => { + const { status, operationResults } = result; +}); + +// After +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph, context) => { + graph.hooks.afterExecuteIterationAsync.tapPromise( + PLUGIN_NAME, + async (status, operationResults, iterationOptions) => { + // Must return the (possibly modified) status + return status; + } + ); +}); +``` + +### `onOperationStatusChanged` → `operationGraph.hooks.onExecutionStatesUpdated` + +The old hook fired once per individual status change. The new hook is **batched**: it fires once per microtask with all changes that occurred within that tick, reducing unnecessary renders or notifications when many operations update simultaneously. + +```typescript +// Before — fires per individual change +hooks.onOperationStatusChanged.tap(PLUGIN_NAME, (record) => { + refreshUI(record); +}); + +// After — fires with a set of all changes in a microtask +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.onExecutionStatesUpdated.tap(PLUGIN_NAME, (changedRecords) => { + for (const record of changedRecords) { + refreshUI(record); + } + }); +}); +``` + +### `beforeExecuteOperation` → `operationGraph.hooks.beforeExecuteOperationAsync` + +The hook has moved to the graph and gained the `Async` suffix to match naming conventions. + +```typescript +// Before +hooks.beforeExecuteOperation.tapPromise(PLUGIN_NAME, async (record) => { + return OperationStatus.FromCache; // short-circuit +}); + +// After +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.beforeExecuteOperationAsync.tapPromise(PLUGIN_NAME, async (record) => { + return OperationStatus.FromCache; // short-circuit + }); +}); +``` + +### `afterExecuteOperation` → `operationGraph.hooks.afterExecuteOperationAsync` + +The hook has moved to the graph and gained the `Async` suffix. + +```typescript +// Before +hooks.afterExecuteOperation.tapPromise(PLUGIN_NAME, async (record) => { }); + +// After +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.afterExecuteOperationAsync.tapPromise(PLUGIN_NAME, async (record) => { }); +}); +``` + +### `createEnvironmentForOperation` → `operationGraph.hooks.createEnvironmentForOperation` + +The hook is now on the graph rather than on `PhasedCommandHooks`. + +```typescript +// Before +hooks.createEnvironmentForOperation.tap(PLUGIN_NAME, (env, record) => { + return { ...env, MY_VAR: 'value' }; +}); + +// After +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.createEnvironmentForOperation.tap(PLUGIN_NAME, (env, record) => { + return { ...env, MY_VAR: 'value' }; + }); +}); +``` + +### `waitingForChanges` → `operationGraph.hooks.onIdle` + +The hook has moved to the graph and been renamed with the standard `on` prefix. + +```typescript +// Before +hooks.waitingForChanges.tap(PLUGIN_NAME, () => { }); + +// After +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.onIdle.tap(PLUGIN_NAME, () => { }); +}); +``` + +### `shutdownAsync` → `IOperationGraph.abortController` + `closeRunnersAsync()` + +The old `shutdownAsync` parallel hook had no direct equivalent. Shutdown is now signalled through the `AbortController` on the graph. To run cleanup logic when the session ends, listen to the abort signal. To shut down long-running runners (e.g. watch-mode IPC processes), call `graph.closeRunnersAsync()`. + +```typescript +// Before +hooks.shutdownAsync.tapPromise(PLUGIN_NAME, async () => { + await myResource.dispose(); +}); + +// After +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.abortController.signal.addEventListener('abort', () => { + void myResource.dispose(); + }, { once: true }); +}); +``` + +--- + +## `ICreateOperationsContext` — removed fields + +Several fields that were on `ICreateOperationsContext` have been removed. Some have direct replacements; others reflect capabilities that no longer exist in the same form. + +| Removed field | Notes | +| --- | --- | +| `isInitial` | Removed. Previously `true` on the first run, `false` on subsequent watch iterations. Because `createOperationsAsync` is now invoked only once per session, the distinction between initial and non-initial is tracked differently — see below. | +| `projectsInUnknownState` | Removed. Previously the set of projects with changed or unknown inputs. This information is now computed per-iteration inside `configureIteration` via `IInputsSnapshot` comparisons. | +| `phaseOriginal` | Removed. Was the pre-expansion set of phases. The watch phases and initial phases are now determined by the command configuration and are not exposed on the context. | +| `invalidateOperation` | Removed. Runners should use `context.getInvalidateCallback()` from `executeAsync`. Plugins can use `IOperationGraph.invalidateOperations()`, available via the graph reference from `onGraphCreatedAsync`. | + +--- + +## Features that are no longer possible + +### Distinguishing initial vs. subsequent runs + +`ICreateOperationsContext` no longer has an `isInitial` flag, and `createOperationsAsync` fires only once — so the hook itself has no iteration-level context at all. The appropriate places to detect first vs. subsequent runs are: + +**In `configureIteration` (plugin hooks):** The second parameter `lastExecutedRecords` is empty on the very first run and non-empty on subsequent ones: + +```typescript +graph.hooks.configureIteration.tap(PLUGIN_NAME, (currentStates, lastExecutedRecords) => { + const isFirstRun = lastExecutedRecords.size === 0; +}); +``` + +**In `IOperationRunner.executeAsync` (custom runners):** The runner is called with a `lastState` parameter. On the first execution `lastState` is `undefined`; on subsequent iterations it holds the `IOperationExecutionResult` from the most recent iteration in which the operation **actually executed to a terminal state**. Runners should use this to decide whether to run an incremental command or a full initial build: + +```typescript +class MyRunner implements IOperationRunner { + async executeAsync(context: IOperationRunnerContext, lastState?: {}): Promise { + if (lastState === undefined) { + // First execution, or the operation was never able to complete in a prior iteration + // (e.g. it was aborted, blocked, or skipped every time) — run full build + } else { + // Operation previously completed — can use incremental strategy + } + // ... + } +} +``` + +This is how `ShellOperationRunner` selects between the `initialCommand` and `incrementalCommand` scripts defined in `rush-project.json`. + +> **Note:** `lastState` is only populated if the operation reached a completed terminal state (`Success`, `SuccessWithWarning`, `Failure`, `FromCache`, or `NoOp`) in a prior iteration. If the previous iteration was aborted before the operation began executing, or if the operation was `Skipped` or `Blocked`, `lastState` will still be `undefined` on the next call. Runners must not assume that a non-`undefined` `lastState` means the previous run succeeded — check `lastState.status` if the prior outcome matters for your incremental logic. +> +> To request re-execution from a long-lived runner, use `context.getInvalidateCallback()` on the `IOperationRunnerContext` to obtain a `(reason: string) => void` callback. This is available from the very first call to `executeAsync`, regardless of whether a previous result exists. + +### Long-lived runners across watch iterations + +Because the same `Operation` objects (and their `runner` instances) are reused for the entire session, custom `IOperationRunner` implementations must be written to handle multiple calls to `executeAsync` on the same instance. A runner that holds external resources — such as a file watcher, a dev server, or a long-running child process — is responsible for managing those resources across iterations. + +Key points for custom runner authors: + +- **`lastState` signals prior completion, not just re-execution.** The `lastState` parameter passed to `executeAsync` is `undefined` on the first call and on any call where the operation did not reach a completed terminal state in the previous iteration (e.g. it was aborted before it began, or was `Skipped`/`Blocked`). It is non-`undefined` only when the operation previously completed with `Success`, `SuccessWithWarning`, `Failure`, `FromCache`, or `NoOp`. Use `lastState` — and `lastState.status` if the prior outcome matters — to decide whether to perform a full or incremental build. + +- **`context.getInvalidateCallback()` for self-invalidation.** Long-lived runners that detect stale outputs (e.g. file watchers, IPC processes) can call `context.getInvalidateCallback()` to obtain a lightweight `(reason: string) => void` callback that requests re-execution. This is available from the very first `executeAsync` call, regardless of whether a previous result exists. Store the returned callback rather than the full context to avoid retaining unnecessary references. + +- **`isActive` tracks background ownership.** If your runner starts a background process (e.g. a dev server) that remains running between iterations, set `isActive = true` for as long as that resource is owned. This allows the dashboard and other tooling to correctly represent the operation's state. + +- **`closeAsync` must be idempotent.** Rush calls `graph.closeRunnersAsync()` when the session ends (on `AbortController` abort). If your runner implements `closeAsync`, it may be called with or without a prior `executeAsync` call, or after the runner has already been closed. Implement it defensively. + +- **State from one iteration does not automatically carry over.** The `IOperationExecutionResult` passed as `lastState` is a snapshot of the previous result. Mutable runner state (e.g. file handles, cached data) must be managed by the runner itself — it is not serialized or restored by Rush. + +```typescript +class MyWatchRunner implements IOperationRunner { + private _server: MyServer | undefined; + + get isActive(): boolean { + return this._server !== undefined; + } + + async executeAsync(context: IOperationRunnerContext, lastState?: {}): Promise { + if (!this._server) { + // First execution — start the server + this._server = await MyServer.startAsync(); + } else { + // Subsequent execution — reload changed files + await this._server.reloadAsync(); + } + return OperationStatus.Success; + } + + async closeAsync(): Promise { + await this._server?.stopAsync(); + this._server = undefined; + } +} +``` + +### Accessing `projectsInUnknownState` from context + +The set of projects with unknown/changed state is no longer pre-computed and passed as context. Plugins that previously consumed `projectsInUnknownState` to decide which operations to run should instead tap `configureIteration` and use the `inputsSnapshot` from `IOperationGraphIterationOptions` to compare state hashes. The built-in logic for this is handled by `PhasedOperationPlugin` and `LegacySkipPlugin`. + +### Parallel shutdown via `shutdownAsync` + +`shutdownAsync` was an `AsyncParallelHook` that allowed multiple plugins to run cleanup concurrently. The replacement via `AbortController` signal listeners achieves a similar result for fire-and-forget cleanup, but there is no built-in mechanism to await all parallel teardown tasks before the process exits. If your plugin needs to perform async cleanup and have it awaited, use `closeRunnersAsync` (for operation runners) or coordinate through the graph's `abortController.signal` and manage your own promises. + +### Mutating the `enabled` state of operations from `createOperations` + +Previously, some plugins disabled operations by mutating the record map returned from `beforeExecuteOperations`. The new model separates graph construction (session-long) from iteration configuration. Operations can only have their `enabled` state changed in `configureIteration` (synchronous) or via `IOperationGraph.setEnabledStates()` at any time. Operations cannot be added or removed from the graph after `createOperationsAsync` completes. + +### `WeightedOperationPlugin` + +This plugin, which assigned operation weights from `rush-project.json` settings, has been removed. Weight assignment is now performed directly by `PhasedOperationPlugin` during graph construction. External plugins that previously called `new WeightedOperationPlugin().apply(hooks)` should remove that call — the behavior is built in. + +--- + +## Accessing session context from graph hooks + +Session-scoped data from `IOperationGraphContext` (build cache config, project selection, phase selection, etc.) is available via closure in the `onGraphCreatedAsync` callback. Store what you need in local variables: + +```typescript +hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph, context) => { + const { buildCacheConfiguration, projectSelection } = context; + + graph.hooks.beforeExecuteIterationAsync.tapPromise(PLUGIN_NAME, async (records, iterationOptions) => { + // buildCacheConfiguration and projectSelection are available here via closure + }); +}); +``` + +## Self-invalidation from runners + +Long-lived runners that need to request re-execution (e.g. a file watcher detecting changes, or an IPC process reporting stale outputs) should call `context.getInvalidateCallback()` on the `IOperationRunnerContext`. This returns a lightweight `(reason: string) => void` callback that delegates to `IOperationGraph.invalidateOperations()` under the hood. Store the returned callback rather than retaining the full context to avoid memory leaks: + +```typescript +class MyWatchRunner implements IOperationRunner { + async executeAsync(context: IOperationRunnerContext, lastState?: {}): Promise { + const invalidate = context.getInvalidateCallback(); + // ... set up watchers that call invalidate('files changed') + return OperationStatus.Success; + } +} +``` + +Because `context` is always available (not just after a completed prior run), runners can obtain the callback on their very first execution. + +> **Note:** If you still need `IOperationGraph` for other purposes (e.g. `setEnabledStates`), the closure pattern via `onGraphCreatedAsync` is still valid. But for simple self-invalidation, prefer `getInvalidateCallback()`. diff --git a/libraries/rush-lib/src/api/CommandLineConfiguration.ts b/libraries/rush-lib/src/api/CommandLineConfiguration.ts index 5fb463764dc..ae639ad56ac 100644 --- a/libraries/rush-lib/src/api/CommandLineConfiguration.ts +++ b/libraries/rush-lib/src/api/CommandLineConfiguration.ts @@ -119,6 +119,13 @@ export interface IPhasedCommandConfig extends IPhasedCommandWithoutPhasesJson, I * How many milliseconds to wait after receiving a file system notification before executing in watch mode. */ watchDebounceMs?: number; + /** + * If true, when running this command in watch mode the operation graph will include every project + * in the repository (respecting phase selection), but only the projects selected by the user's + * CLI project selection parameters will be initially enabled. Other projects will remain disabled + * unless they become required or are explicitly selected in a subsequent pass. + */ + includeAllProjectsInWatchGraph?: boolean; /** * If set to `true`, then this phased command will always perform an install before executing, regardless of CLI flags. * If set to `false`, then Rush will define a built-in "--install" CLI flag for this command. @@ -383,6 +390,8 @@ export class CommandLineConfiguration { if (watchOptions) { normalizedCommand.alwaysWatch = watchOptions.alwaysWatch; normalizedCommand.watchDebounceMs = watchOptions.debounceMs; + normalizedCommand.includeAllProjectsInWatchGraph = + !!watchOptions.includeAllProjectsInWatchGraph; // No implicit phase dependency expansion for watch mode. for (const phaseName of watchOptions.watchPhases) { diff --git a/libraries/rush-lib/src/api/CommandLineJson.ts b/libraries/rush-lib/src/api/CommandLineJson.ts index e6507e49633..8991f5996f3 100644 --- a/libraries/rush-lib/src/api/CommandLineJson.ts +++ b/libraries/rush-lib/src/api/CommandLineJson.ts @@ -52,6 +52,7 @@ export interface IPhasedCommandJson extends IPhasedCommandWithoutPhasesJson { alwaysWatch: boolean; debounceMs?: number; watchPhases: string[]; + includeAllProjectsInWatchGraph?: boolean; }; installOptions?: { alwaysInstall: boolean; diff --git a/libraries/rush-lib/src/api/EventHooks.ts b/libraries/rush-lib/src/api/EventHooks.ts index 8d671c50b38..d53fdd365ad 100644 --- a/libraries/rush-lib/src/api/EventHooks.ts +++ b/libraries/rush-lib/src/api/EventHooks.ts @@ -6,7 +6,7 @@ import { Enum } from '@rushstack/node-core-library'; import type { IEventHooksJson } from './RushConfiguration'; /** - * Events happen during Rush runs. + * Events happen during Rush invocation. * @beta */ export enum Event { diff --git a/libraries/rush-lib/src/cli/RushCommandLineParser.ts b/libraries/rush-lib/src/cli/RushCommandLineParser.ts index cce1a59d91c..67750429c1f 100644 --- a/libraries/rush-lib/src/cli/RushCommandLineParser.ts +++ b/libraries/rush-lib/src/cli/RushCommandLineParser.ts @@ -490,6 +490,7 @@ export class RushCommandLineParser extends CommandLineParser { originalPhases: command.originalPhases, watchPhases: command.watchPhases, watchDebounceMs: command.watchDebounceMs ?? RushConstants.defaultWatchDebounceMs, + includeAllProjectsInWatchGraph: command.includeAllProjectsInWatchGraph || false, phases: commandLineConfiguration.phases, alwaysWatch: command.alwaysWatch, diff --git a/libraries/rush-lib/src/cli/parsing/ParseParallelism.ts b/libraries/rush-lib/src/cli/parsing/ParseParallelism.ts deleted file mode 100644 index 2cd16906eb8..00000000000 --- a/libraries/rush-lib/src/cli/parsing/ParseParallelism.ts +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import * as os from 'node:os'; - -import { IS_WINDOWS } from '../../utilities/executionUtilities'; - -export function getNumberOfCores(): number { - return os.availableParallelism?.() ?? os.cpus().length; -} - -/** - * Since the JSON value is a string, it must be a percentage like "50%", - * which we convert to a number based on the available parallelism. - * For example, if the available parallelism (not the -p flag) is 8 and the weight is "50%", - * then the resulting weight will be 4. - * - * @param weight - * @returns - */ -export function parseParallelismPercent(weight: string, numberOfCores: number = getNumberOfCores()): number { - const percentageRegExp: RegExp = /^\d+(\.\d+)?%$/; - - if (!percentageRegExp.test(weight)) { - throw new Error(`Expecting a percentage string like "12%" or "34.56%".`); - } - - const percentValue: number = parseFloat(weight.slice(0, -1)); - - if (percentValue <= 0) { - throw new Error(`Invalid percentage value of "${percentValue}": value must be greater than zero`); - } - - if (percentValue > 100) { - throw new Error(`Invalid percentage value of "${percentValue}": value must not exceed 100%`); - } - - // Use as much CPU as possible, so we round down the weight here - return Math.max(1, Math.floor((percentValue / 100) * numberOfCores)); -} - -/** - * Parses a command line specification for desired parallelism. - * Factored out to enable unit tests - */ -export function parseParallelism( - rawParallelism: string | undefined, - numberOfCores: number = getNumberOfCores() -): number { - if (rawParallelism) { - rawParallelism = rawParallelism.trim(); - - if (rawParallelism === 'max') { - return numberOfCores; - } - - if (rawParallelism.endsWith('%')) { - return parseParallelismPercent(rawParallelism, numberOfCores); - } - - const parallelismAsNumber: number = Number(rawParallelism); - if (!isNaN(parallelismAsNumber)) { - return Math.max(parallelismAsNumber, 1); - } - - throw new Error( - `Invalid parallelism value of "${rawParallelism}": expected a number, a percentage string, or "max"` - ); - } else { - // If an explicit parallelism number wasn't provided, then choose a sensible - // default. - if (IS_WINDOWS) { - // On desktop Windows, some people have complained that their system becomes - // sluggish if Rush is using all the CPU cores. Leave one thread for - // other operations. For CI environments, you can use the "max" argument to use all available cores. - return Math.max(numberOfCores - 1, 1); - } else { - // Unix-like operating systems have more balanced scheduling, so default - // to the number of CPU cores - return numberOfCores; - } - } -} diff --git a/libraries/rush-lib/src/cli/parsing/SelectionParameterSet.ts b/libraries/rush-lib/src/cli/parsing/SelectionParameterSet.ts index c8d90bf1a1b..758fbb5737b 100644 --- a/libraries/rush-lib/src/cli/parsing/SelectionParameterSet.ts +++ b/libraries/rush-lib/src/cli/parsing/SelectionParameterSet.ts @@ -241,7 +241,10 @@ export class SelectionParameterSet { * * If no parameters are specified, returns all projects in the Rush config file. */ - public async getSelectedProjectsAsync(terminal: ITerminal): Promise> { + public async getSelectedProjectsAsync( + terminal: ITerminal, + allowEmptySelection?: boolean + ): Promise> { // Hack out the old version-policy parameters for (const value of this._fromVersionPolicy.values) { (this._fromProject.values as string[]).push(`version-policy:${value}`); @@ -266,7 +269,7 @@ export class SelectionParameterSet { // If no selection parameters are specified, return everything if (!isSelectionSpecified) { - return new Set(this._rushConfiguration.projects); + return allowEmptySelection ? new Set() : new Set(this._rushConfiguration.projects); } const [ diff --git a/libraries/rush-lib/src/cli/parsing/test/ParseParallelism.test.ts b/libraries/rush-lib/src/cli/parsing/test/ParseParallelism.test.ts deleted file mode 100644 index 7d2315d3d8a..00000000000 --- a/libraries/rush-lib/src/cli/parsing/test/ParseParallelism.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { parseParallelism } from '../ParseParallelism'; - -describe(parseParallelism.name, () => { - it('throwsErrorOnInvalidParallelism', () => { - expect(() => parseParallelism('tequila')).toThrowErrorMatchingSnapshot(); - }); - - it('createsWithPercentageBasedParallelism', () => { - const value: number = parseParallelism('50%', 20); - expect(value).toEqual(10); - }); - - it('throwsErrorOnInvalidParallelismPercentage', () => { - expect(() => parseParallelism('200%')).toThrowErrorMatchingSnapshot(); - }); -}); diff --git a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts index e75efcbeaf5..38384f811fb 100644 --- a/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts +++ b/libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import { once } from 'node:events'; + import type { AsyncSeriesHook } from 'tapable'; import { AlreadyReportedError } from '@rushstack/node-core-library'; -import { type ITerminal, Terminal, Colorize } from '@rushstack/terminal'; +import { type ITerminal, Terminal, Colorize, StdioWritable } from '@rushstack/terminal'; import type { CommandLineFlagParameter, CommandLineParameter, @@ -14,17 +16,16 @@ import type { import type { Subspace } from '../../api/Subspace'; import type { IPhasedCommand } from '../../pluginFramework/RushLifeCycle'; import { + type IOperationGraphContext, PhasedCommandHooks, - type ICreateOperationsContext, - type IExecuteOperationsContext + type ICreateOperationsContext } from '../../pluginFramework/PhasedCommandHooks'; +import type { IOperationGraphIterationOptions } from '../../logic/operations/IOperationGraph'; import { SetupChecks } from '../../logic/SetupChecks'; -import { Stopwatch, StopwatchState } from '../../utilities/Stopwatch'; +import { Stopwatch } from '../../utilities/Stopwatch'; import { BaseScriptAction, type IBaseScriptActionOptions } from './BaseScriptAction'; -import { - type IOperationExecutionManagerOptions, - OperationExecutionManager -} from '../../logic/operations/OperationExecutionManager'; +import type { IOperationGraphOptions, IOperationGraphTelemetry } from '../../logic/operations/OperationGraph'; +import { OperationGraph } from '../../logic/operations/OperationGraph'; import { RushConstants } from '../../logic/RushConstants'; import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; @@ -32,20 +33,20 @@ import { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import { SelectionParameterSet } from '../parsing/SelectionParameterSet'; import type { IPhase, IPhasedCommandConfig } from '../../api/CommandLineConfiguration'; import type { Operation } from '../../logic/operations/Operation'; -import type { OperationExecutionRecord } from '../../logic/operations/OperationExecutionRecord'; import { associateParametersByPhase } from '../parsing/associateParametersByPhase'; import { PhasedOperationPlugin } from '../../logic/operations/PhasedOperationPlugin'; import { ShellOperationRunnerPlugin } from '../../logic/operations/ShellOperationRunnerPlugin'; import { Event } from '../../api/EventHooks'; import { ProjectChangeAnalyzer } from '../../logic/ProjectChangeAnalyzer'; import { OperationStatus } from '../../logic/operations/OperationStatus'; -import type { - IExecutionResult, - IOperationExecutionResult -} from '../../logic/operations/IOperationExecutionResult'; +import type { IExecutionResult } from '../../logic/operations/IOperationExecutionResult'; import { OperationResultSummarizerPlugin } from '../../logic/operations/OperationResultSummarizerPlugin'; -import type { ITelemetryData, ITelemetryOperationResult } from '../../logic/Telemetry'; -import { parseParallelism } from '../parsing/ParseParallelism'; +import type { ITelemetryData } from '../../logic/Telemetry'; +import { + getNumberOfCores, + parseParallelism, + type Parallelism +} from '../../logic/operations/ParseParallelism'; import { CobuildConfiguration } from '../../api/CobuildConfiguration'; import { CacheableOperationPlugin } from '../../logic/operations/CacheableOperationPlugin'; import type { IInputsSnapshot, GetInputsSnapshotAsyncFn } from '../../logic/incremental/InputsSnapshot'; @@ -53,9 +54,7 @@ import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import { LegacySkipPlugin } from '../../logic/operations/LegacySkipPlugin'; import { ValidateOperationsPlugin } from '../../logic/operations/ValidateOperationsPlugin'; import { ShardedPhasedOperationPlugin } from '../../logic/operations/ShardedPhaseOperationPlugin'; -import type { ProjectWatcher } from '../../logic/ProjectWatcher'; import { FlagFile } from '../../api/FlagFile'; -import { WeightedOperationPlugin } from '../../logic/operations/WeightedOperationPlugin'; import { getVariantAsync, VARIANT_PARAMETER } from '../../api/Variants'; import { Selection } from '../../logic/Selection'; import { NodeDiagnosticDirPlugin } from '../../logic/operations/NodeDiagnosticDirPlugin'; @@ -77,6 +76,7 @@ export interface IPhasedScriptActionOptions extends IBaseScriptActionOptions; initialPhases: Set; watchPhases: Set; + includeAllProjectsInWatchGraph: boolean; phases: Map; alwaysWatch: boolean; @@ -85,46 +85,14 @@ export interface IPhasedScriptActionOptions extends IBaseScriptActionOptions; - initialCreateOperationsContext: ICreateOperationsContext; - stopwatch: Stopwatch; - terminal: ITerminal; -} - -interface IRunPhasesOptions extends IInitialRunPhasesOptions { - getInputsSnapshotAsync: GetInputsSnapshotAsyncFn | undefined; - initialSnapshot: IInputsSnapshot | undefined; - executionManagerOptions: IOperationExecutionManagerOptions; -} - interface IExecuteOperationsOptions { - executeOperationsContext: IExecuteOperationsContext; - executionManagerOptions: IOperationExecutionManagerOptions; + graph: OperationGraph; ignoreHooks: boolean; - operations: Set; + isWatch: boolean; stopwatch: Stopwatch; terminal: ITerminal; } -interface IPhasedCommandTelemetry { - [key: string]: string | number | boolean; - isInitial: boolean; - isWatch: boolean; - - countAll: number; - countSuccess: number; - countSuccessWithWarnings: number; - countFailure: number; - countBlocked: number; - countFromCache: number; - countSkipped: number; - countNoOp: number; -} - /** * This class implements phased commands which are run individually for each project in the repo, * possibly in parallel, and which may define multiple phases. @@ -152,10 +120,9 @@ export class PhasedScriptAction extends BaseScriptAction i private readonly _watchDebounceMs: number; private readonly _alwaysWatch: boolean; private readonly _alwaysInstall: boolean | undefined; + private readonly _includeAllProjectsInWatchGraph: boolean; private readonly _knownPhases: ReadonlyMap; private readonly _terminal: ITerminal; - private _changedProjectsOnly: boolean; - private _executionAbortController: AbortController | undefined; private readonly _changedProjectsOnlyParameter: CommandLineFlagParameter | undefined; private readonly _selectionParameters: SelectionParameterSet; @@ -184,19 +151,10 @@ export class PhasedScriptAction extends BaseScriptAction i this._watchDebounceMs = options.watchDebounceMs ?? RushConstants.defaultWatchDebounceMs; this._alwaysWatch = options.alwaysWatch; this._alwaysInstall = options.alwaysInstall; + this._includeAllProjectsInWatchGraph = options.includeAllProjectsInWatchGraph; this._runsBeforeInstall = false; this._knownPhases = options.phases; - this._changedProjectsOnly = false; this.sessionAbortController = new AbortController(); - this._executionAbortController = undefined; - - this.sessionAbortController.signal.addEventListener( - 'abort', - () => { - this._executionAbortController?.abort(); - }, - { once: true } - ); this.hooks = new PhasedCommandHooks(); @@ -335,7 +293,8 @@ export class PhasedScriptAction extends BaseScriptAction i } public async runAsync(): Promise { - const stopwatch: Stopwatch = Stopwatch.start(); + // Initialize the stopwatch's start time at 0 (process startup). + const stopwatch: Stopwatch = Stopwatch.start(0); if (this._alwaysInstall || this._installParameter?.value) { await measureAsyncFn(`${PERF_PREFIX}:install`, async () => { @@ -393,12 +352,11 @@ export class PhasedScriptAction extends BaseScriptAction i // if this is parallelizable, then use the value from the flag (undefined or a number), // if parallelism is not enabled, then restrict to 1 core - const parallelism: number = this._enableParallelism + const maxParallelism: number = getNumberOfCores(); + const parallelism: Parallelism = this._enableParallelism ? parseParallelism(this._parallelismParameter?.value) : 1; - const includePhaseDeps: boolean = this._includePhaseDeps?.value ?? false; - await measureAsyncFn(`${PERF_PREFIX}:applyStandardPlugins`, async () => { // Generates the default operation graph new PhasedOperationPlugin().apply(hooks); @@ -406,8 +364,7 @@ export class PhasedScriptAction extends BaseScriptAction i new ShardedPhasedOperationPlugin().apply(hooks); // Applies the Shell Operation Runner to selected operations new ShellOperationRunnerPlugin().apply(hooks); - - new WeightedOperationPlugin().apply(hooks); + // Verifies correctness of rush-project.json entries for the graph new ValidateOperationsPlugin(terminal).apply(hooks); // Forward ignored parameters to child processes as an environment variable @@ -455,7 +412,6 @@ export class PhasedScriptAction extends BaseScriptAction i const isQuietMode: boolean = !this._verboseParameter.value; const changedProjectsOnly: boolean = !!this._changedProjectsOnlyParameter?.value; - this._changedProjectsOnly = changedProjectsOnly; let buildCacheConfiguration: BuildCacheConfiguration | undefined; let cobuildConfiguration: CobuildConfiguration | undefined; @@ -463,33 +419,38 @@ export class PhasedScriptAction extends BaseScriptAction i await measureAsyncFn(`${PERF_PREFIX}:configureBuildCache`, async () => { [buildCacheConfiguration, cobuildConfiguration] = await Promise.all([ BuildCacheConfiguration.tryLoadAsync(terminal, this.rushConfiguration, this.rushSession), - CobuildConfiguration.tryLoadAsync(terminal, this.rushConfiguration, this.rushSession) + CobuildConfiguration.tryLoadAsync(terminal, this.rushConfiguration, this.rushSession).then( + async (cobuildCfg: CobuildConfiguration | undefined) => { + if (cobuildCfg) { + await cobuildCfg.createLockProviderAsync(terminal); + } + return cobuildCfg; + } + ) ]); - if (cobuildConfiguration) { - await cobuildConfiguration.createLockProviderAsync(terminal); - } }); } + const isWatch: boolean = this._watchParameter?.value || this._alwaysWatch; + const generateFullGraph: boolean = isWatch && this._includeAllProjectsInWatchGraph; + try { const projectSelection: Set = await measureAsyncFn( `${PERF_PREFIX}:getSelectedProjects`, - () => this._selectionParameters.getSelectedProjectsAsync(terminal) + () => this._selectionParameters.getSelectedProjectsAsync(terminal, generateFullGraph) ); - if (!projectSelection.size) { - terminal.writeLine( - Colorize.yellow(`The command line selection parameters did not match any projects.`) - ); - return; - } - const customParametersByName: Map = new Map(); for (const [configParameter, parserParameter] of this.customParameters) { customParametersByName.set(configParameter.longName, parserParameter); } - const isWatch: boolean = this._watchParameter?.value || this._alwaysWatch; + if (!generateFullGraph && !projectSelection.size) { + terminal.writeLine( + Colorize.yellow(`The command line selection parameters did not match any projects.`) + ); + return; + } await measureAsyncFn(`${PERF_PREFIX}:applySituationalPlugins`, async () => { if (isWatch && this._noIPCParameter?.value === false) { @@ -551,8 +512,9 @@ export class PhasedScriptAction extends BaseScriptAction i } }); - const relevantProjects: Set = - Selection.expandAllDependencies(projectSelection); + const relevantProjects: Set = generateFullGraph + ? new Set(this.rushConfiguration.projects) + : Selection.expandAllDependencies(projectSelection); const projectConfigurations: ReadonlyMap = this ._runsBeforeInstall @@ -561,392 +523,160 @@ export class PhasedScriptAction extends BaseScriptAction i RushProjectConfiguration.tryLoadForProjectsAsync(relevantProjects, terminal) ); - const initialCreateOperationsContext: ICreateOperationsContext = { + const includePhaseDeps: boolean = this._includePhaseDeps?.value ?? false; + + const createOperationsContext: ICreateOperationsContext = { buildCacheConfiguration, - changedProjectsOnly, cobuildConfiguration, customParameters: customParametersByName, + changedProjectsOnly, + includePhaseDeps, isIncrementalBuildAllowed: this._isIncrementalBuildAllowed, - isInitial: true, isWatch, rushConfiguration: this.rushConfiguration, parallelism, - phaseOriginal: new Set(this._originalPhases), - phaseSelection: new Set(this._initialPhases), - includePhaseDeps, + phaseSelection: isWatch + ? this._watchPhases + : includePhaseDeps + ? this._originalPhases + : this._initialPhases, projectSelection, - projectConfigurations, - projectsInUnknownState: projectSelection + generateFullGraph, + projectConfigurations }; - const executionManagerOptions: Omit< - IOperationExecutionManagerOptions, - 'beforeExecuteOperations' | 'inputsSnapshot' - > = { + const operations: Set = await measureAsyncFn(`${PERF_PREFIX}:createOperations`, () => + this.hooks.createOperationsAsync.promise(new Set(), createOperationsContext) + ); + + const [getInputsSnapshotAsync, initialSnapshot] = await measureAsyncFn( + `${PERF_PREFIX}:analyzeRepoState`, + async () => { + terminal.write('Analyzing repo state... '); + const repoStateStopwatch: Stopwatch = new Stopwatch(); + repoStateStopwatch.start(); + + const analyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); + const innerGetInputsSnapshotAsync: GetInputsSnapshotAsyncFn | undefined = + await analyzer._tryGetSnapshotProviderAsync( + projectConfigurations, + terminal, + // We need to include all dependencies, otherwise build cache id calculation will be incorrect + relevantProjects + ); + const innerInitialSnapshot: IInputsSnapshot | undefined = innerGetInputsSnapshotAsync + ? await innerGetInputsSnapshotAsync() + : undefined; + + repoStateStopwatch.stop(); + terminal.writeLine(`DONE (${repoStateStopwatch.toString()})`); + terminal.writeLine(); + return [innerGetInputsSnapshotAsync, innerInitialSnapshot]; + } + ); + + let executionTelemetryHandler: IOperationGraphTelemetry | undefined; + const { telemetry: parserTelemetry } = this.parser; + if (parserTelemetry) { + const { _changedProjectsOnlyParameter: changedProjectsOnlyParameter } = this; + executionTelemetryHandler = { + changedProjectsOnlyKey: + changedProjectsOnlyParameter?.scopedLongName ?? changedProjectsOnlyParameter?.longName, + initialExtraData: { + // Fields preserved across the command invocation + ...this._selectionParameters.getTelemetry(), + ...this.getParameterStringMap() + }, + nameForLog: this.actionName, + log: (logEntry: ITelemetryData) => { + parserTelemetry.log(logEntry); + parserTelemetry.flush(); + } + }; + } + + const graphOptions: IOperationGraphOptions = { quietMode: isQuietMode, debugMode: this.parser.isDebug, + destinations: [StdioWritable.instance], parallelism, + maxParallelism, allowOversubscription: this._allowOversubscription, - beforeExecuteOperationAsync: async (record: OperationExecutionRecord) => { - return await this.hooks.beforeExecuteOperation.promise(record); - }, - afterExecuteOperationAsync: async (record: OperationExecutionRecord) => { - await this.hooks.afterExecuteOperation.promise(record); - }, - createEnvironmentForOperation: this.hooks.createEnvironmentForOperation.isUsed() - ? (record: OperationExecutionRecord) => { - return this.hooks.createEnvironmentForOperation.call({ ...process.env }, record); - } - : undefined, - onOperationStatusChangedAsync: (record: OperationExecutionRecord) => { - this.hooks.onOperationStatusChanged.call(record); - } + isWatch, + pauseNextIteration: false, + getInputsSnapshotAsync, + abortController: this.sessionAbortController, + telemetry: executionTelemetryHandler + }; + + const graph: OperationGraph = new OperationGraph(operations, graphOptions); + + const graphContext: IOperationGraphContext = { + ...createOperationsContext, + initialSnapshot }; - const initialInternalOptions: IInitialRunPhasesOptions = { - initialCreateOperationsContext, - executionManagerOptions, + const abortPromise: Promise = once(this.sessionAbortController.signal, 'abort').then(async () => { + terminal.writeLine(`Shutting down Rush...`); + return await graph.abortCurrentIterationAsync(); + }); + + await measureAsyncFn(`${PERF_PREFIX}:executionManager`, async () => { + await hooks.onGraphCreatedAsync.promise(graph, graphContext); + }); + + const executeOptions: IExecuteOperationsOptions = { + graph, + ignoreHooks: !!this._ignoreHooksParameter.value, + isWatch, stopwatch, terminal }; - const internalOptions: IRunPhasesOptions = await measureAsyncFn(`${PERF_PREFIX}:runInitialPhases`, () => - this._runInitialPhasesAsync(initialInternalOptions) - ); - + const initialIterationOptions: IOperationGraphIterationOptions = { + inputsSnapshot: initialSnapshot + }; if (isWatch) { + if (!initialSnapshot) { + terminal.writeErrorLine(`Unable to run in watch mode: could not analyze repository state`); + throw new AlreadyReportedError(); + } + if (buildCacheConfiguration) { // Cache writes are not supported during watch mode, only reads. buildCacheConfiguration.cacheWriteEnabled = false; } - await this._runWatchPhasesAsync(internalOptions); - terminal.writeDebugLine(`Watch mode exited.`); - } - } finally { - if (cobuildConfiguration) { - await cobuildConfiguration.destroyLockProviderAsync(); - } - } - } - - private async _runInitialPhasesAsync(options: IInitialRunPhasesOptions): Promise { - const { - initialCreateOperationsContext, - executionManagerOptions: partialExecutionManagerOptions, - stopwatch, - terminal - } = options; - - const { projectConfigurations } = initialCreateOperationsContext; - const { projectSelection } = initialCreateOperationsContext; - - const operations: Set = await measureAsyncFn(`${PERF_PREFIX}:createOperations`, () => - this.hooks.createOperations.promise(new Set(), initialCreateOperationsContext) - ); - - const [getInputsSnapshotAsync, initialSnapshot] = await measureAsyncFn( - `${PERF_PREFIX}:analyzeRepoState`, - async () => { - terminal.write('Analyzing repo state... '); - const repoStateStopwatch: Stopwatch = new Stopwatch(); - repoStateStopwatch.start(); - - const analyzer: ProjectChangeAnalyzer = new ProjectChangeAnalyzer(this.rushConfiguration); - const innerGetInputsSnapshotAsync: GetInputsSnapshotAsyncFn | undefined = - await analyzer._tryGetSnapshotProviderAsync( - projectConfigurations, - terminal, - // We need to include all dependencies, otherwise build cache id calculation will be incorrect - Selection.expandAllDependencies(projectSelection) - ); - const innerInitialSnapshot: IInputsSnapshot | undefined = innerGetInputsSnapshotAsync - ? await innerGetInputsSnapshotAsync() - : undefined; - - repoStateStopwatch.stop(); - terminal.writeLine(`DONE (${repoStateStopwatch.toString()})`); - terminal.writeLine(); - return [innerGetInputsSnapshotAsync, innerInitialSnapshot]; - } - ); - - const abortController: AbortController = (this._executionAbortController = new AbortController()); - const initialExecuteOperationsContext: IExecuteOperationsContext = { - ...initialCreateOperationsContext, - inputsSnapshot: initialSnapshot, - abortController - }; - - const executionManagerOptions: IOperationExecutionManagerOptions = { - ...partialExecutionManagerOptions, - inputsSnapshot: initialSnapshot, - beforeExecuteOperationsAsync: async (records: Map) => { - await measureAsyncFn(`${PERF_PREFIX}:beforeExecuteOperations`, () => - this.hooks.beforeExecuteOperations.promise(records, initialExecuteOperationsContext) + const { ProjectWatcher } = await import( + /* webpackChunkName: 'ProjectWatcher' */ + '../../logic/ProjectWatcher' ); - } - }; - - const initialOptions: IExecuteOperationsOptions = { - executeOperationsContext: initialExecuteOperationsContext, - ignoreHooks: false, - operations, - stopwatch, - executionManagerOptions, - terminal - }; - - await measureAsyncFn(`${PERF_PREFIX}:executeOperations`, () => - this._executeOperationsAsync(initialOptions) - ); - - return { - ...options, - executionManagerOptions, - getInputsSnapshotAsync, - initialSnapshot - }; - } - - private _registerWatchModeInterface(projectWatcher: ProjectWatcher): void { - const buildOnceKey: 'b' = 'b'; - const changedProjectsOnlyKey: 'c' = 'c'; - const invalidateKey: 'i' = 'i'; - const quitKey: 'q' = 'q'; - const abortKey: 'a' = 'a'; - const toggleWatcherKey: 'w' = 'w'; - const shutdownProcessesKey: 'x' = 'x'; - - const terminal: ITerminal = this._terminal; - - projectWatcher.setPromptGenerator((isPaused: boolean) => { - const promptLines: string[] = [ - ` Press <${quitKey}> to gracefully exit.`, - ` Press <${abortKey}> to abort queued operations. Any that have started will finish.`, - ` Press <${toggleWatcherKey}> to ${isPaused ? 'resume' : 'pause'}.`, - ` Press <${invalidateKey}> to invalidate all projects.`, - ` Press <${changedProjectsOnlyKey}> to ${ - this._changedProjectsOnly ? 'disable' : 'enable' - } changed-projects-only mode (${this._changedProjectsOnly ? 'ENABLED' : 'DISABLED'}).` - ]; - if (isPaused) { - promptLines.push(` Press <${buildOnceKey}> to build once.`); - } - if (this._noIPCParameter?.value === false) { - promptLines.push(` Press <${shutdownProcessesKey}> to reset child processes.`); - } - return promptLines; - }); - - const onKeyPress = (key: string): void => { - switch (key) { - case quitKey: - terminal.writeLine(`Exiting watch mode and aborting any scheduled work...`); - process.stdin.setRawMode(false); - process.stdin.off('data', onKeyPress); - process.stdin.unref(); - this.sessionAbortController.abort(); - break; - case abortKey: - terminal.writeLine(`Aborting current iteration...`); - this._executionAbortController?.abort(); - break; - case toggleWatcherKey: - if (projectWatcher.isPaused) { - projectWatcher.resume(); - } else { - projectWatcher.pause(); - } - break; - case buildOnceKey: - if (projectWatcher.isPaused) { - projectWatcher.clearStatus(); - terminal.writeLine(`Building once...`); - projectWatcher.resume(); - projectWatcher.pause(); - } - break; - case invalidateKey: - projectWatcher.clearStatus(); - terminal.writeLine(`Invalidating all operations...`); - projectWatcher.invalidateAll('manual trigger'); - if (!projectWatcher.isPaused) { - projectWatcher.resume(); - } - break; - case changedProjectsOnlyKey: - this._changedProjectsOnly = !this._changedProjectsOnly; - projectWatcher.rerenderStatus(); - break; - case shutdownProcessesKey: - projectWatcher.clearStatus(); - terminal.writeLine(`Shutting down long-lived child processes...`); - // TODO: Inject this promise into the execution queue somewhere so that it gets waited on between runs - void this.hooks.shutdownAsync.promise(); - break; - case '\u0003': - process.stdin.setRawMode(false); - process.stdin.off('data', onKeyPress); - process.stdin.unref(); - this.sessionAbortController.abort(); - process.kill(process.pid, 'SIGINT'); - break; - } - }; - - process.stdin.setRawMode(true); - process.stdin.resume(); - process.stdin.setEncoding('utf8'); - process.stdin.on('data', onKeyPress); - } - - /** - * Runs the command in watch mode. Fundamentally is a simple loop: - * 1) Wait for a change to one or more projects in the selection - * 2) Invoke the command on the changed projects, and, if applicable, impacted projects - * Uses the same algorithm as --impacted-by - * 3) Goto (1) - */ - private async _runWatchPhasesAsync(options: IRunPhasesOptions): Promise { - const { - getInputsSnapshotAsync, - initialSnapshot, - initialCreateOperationsContext, - executionManagerOptions, - stopwatch, - terminal - } = options; - - const phaseOriginal: Set = new Set(this._watchPhases); - const phaseSelection: Set = new Set(this._watchPhases); - - const { projectSelection: projectsToWatch } = initialCreateOperationsContext; - - if (!getInputsSnapshotAsync || !initialSnapshot) { - terminal.writeErrorLine( - `Cannot watch for changes if the Rush repo is not in a Git repository, exiting.` - ); - throw new AlreadyReportedError(); - } - - // Use async import so that we don't pay the cost for sync builds - const { ProjectWatcher } = await import( - /* webpackChunkName: 'ProjectWatcher' */ - '../../logic/ProjectWatcher' - ); - - const sessionAbortController: AbortController = this.sessionAbortController; - const abortSignal: AbortSignal = sessionAbortController.signal; - - const projectWatcher: typeof ProjectWatcher.prototype = new ProjectWatcher({ - getInputsSnapshotAsync, - initialSnapshot, - debounceMs: this._watchDebounceMs, - rushConfiguration: this.rushConfiguration, - projectsToWatch, - abortSignal, - terminal - }); - - // Ensure process.stdin allows interactivity before using TTY-only APIs - if (process.stdin.isTTY) { - this._registerWatchModeInterface(projectWatcher); - } - - const onWaitingForChanges = (): void => { - // Allow plugins to display their own messages when waiting for changes. - this.hooks.waitingForChanges.call(); - - // Report so that the developer can always see that it is in watch mode as the latest console line. - terminal.writeLine( - `Watching for changes to ${projectsToWatch.size} ${ - projectsToWatch.size === 1 ? 'project' : 'projects' - }. Press Ctrl+C to exit.` - ); - }; - - function invalidateOperation(operation: Operation, reason: string): void { - const { associatedProject } = operation; - // Since ProjectWatcher only tracks entire projects, widen the operation to its project - // Revisit when migrating to @rushstack/operation-graph and we have a long-lived operation graph - projectWatcher.invalidateProject(associatedProject, `${operation.name!} (${reason})`); - } - - // Loop until Ctrl+C - while (!abortSignal.aborted) { - // On the initial invocation, this promise will return immediately with the full set of projects - const { changedProjects, inputsSnapshot: state } = await measureAsyncFn( - `${PERF_PREFIX}:waitForChanges`, - () => projectWatcher.waitForChangeAsync(onWaitingForChanges) - ); - - if (abortSignal.aborted) { - return; - } - - if (stopwatch.state === StopwatchState.Stopped) { - // Clear and reset the stopwatch so that we only report time from a single execution at a time - stopwatch.reset(); - stopwatch.start(); - } - - terminal.writeLine( - `Detected changes in ${changedProjects.size} project${changedProjects.size === 1 ? '' : 's'}:` - ); - const names: string[] = [...changedProjects].map((x) => x.packageName).sort(); - for (const name of names) { - terminal.writeLine(` ${Colorize.cyan(name)}`); - } - - const initialAbortController: AbortController = (this._executionAbortController = - new AbortController()); - - // Account for consumer relationships - const executeOperationsContext: IExecuteOperationsContext = { - ...initialCreateOperationsContext, - abortController: initialAbortController, - changedProjectsOnly: !!this._changedProjectsOnly, - isInitial: false, - inputsSnapshot: state, - projectsInUnknownState: changedProjects, - phaseOriginal, - phaseSelection, - invalidateOperation - }; + const watcher: typeof ProjectWatcher.prototype = new ProjectWatcher({ + rushConfiguration: this.rushConfiguration, + graph, + initialSnapshot, + terminal, + debounceMs: this._watchDebounceMs + }); + watcher.clearStatus(); - const operations: Set = await measureAsyncFn(`${PERF_PREFIX}:createOperations`, () => - this.hooks.createOperations.promise(new Set(), executeOperationsContext) - ); + await measureAsyncFn(`${PERF_PREFIX}:executeOperationsInner`, async () => { + return await graph.executeAsync(initialIterationOptions); + }); - const executeOptions: IExecuteOperationsOptions = { - executeOperationsContext, - // For now, don't run pre-build or post-build in watch mode - ignoreHooks: true, - operations, - stopwatch, - executionManagerOptions: { - ...executionManagerOptions, - inputsSnapshot: state, - beforeExecuteOperationsAsync: async (records: Map) => { - await measureAsyncFn(`${PERF_PREFIX}:beforeExecuteOperations`, () => - this.hooks.beforeExecuteOperations.promise(records, executeOperationsContext) - ); - } - }, - terminal - }; + await abortPromise; - try { - // Delegate the the underlying command, for only the projects that need reprocessing - await measureAsyncFn(`${PERF_PREFIX}:executeOperations`, () => - this._executeOperationsAsync(executeOptions) + terminal.writeLine(`Watch mode exited.`); + } else { + await measureAsyncFn(`${PERF_PREFIX}:runInitialPhases`, () => + measureAsyncFn(`${PERF_PREFIX}:executeOperations`, () => + this._executeOperationsAsync(executeOptions, initialIterationOptions) + ) ); - } catch (err) { - // In watch mode, we want to rebuild even if the original build failed. - if (!(err instanceof AlreadyReportedError)) { - throw err; - } + } + } finally { + if (cobuildConfiguration) { + await cobuildConfiguration.destroyLockProviderAsync(); } } } @@ -954,22 +684,11 @@ export class PhasedScriptAction extends BaseScriptAction i /** * Runs a set of operations and reports the results. */ - private async _executeOperationsAsync(options: IExecuteOperationsOptions): Promise { - const { - executeOperationsContext, - executionManagerOptions, - ignoreHooks, - operations, - stopwatch, - terminal - } = options; - - const executionManager: OperationExecutionManager = new OperationExecutionManager( - operations, - executionManagerOptions - ); - - const { isInitial, isWatch, abortController, invalidateOperation } = executeOperationsContext; + private async _executeOperationsAsync( + options: IExecuteOperationsOptions, + iterationOptions: IOperationGraphIterationOptions + ): Promise { + const { graph, ignoreHooks, stopwatch, terminal } = options; let success: boolean = false; let result: IExecutionResult | undefined; @@ -977,15 +696,13 @@ export class PhasedScriptAction extends BaseScriptAction i try { const definiteResult: IExecutionResult = await measureAsyncFn( `${PERF_PREFIX}:executeOperationsInner`, - () => executionManager.executeAsync(abortController) + async () => { + return await graph.executeAsync(iterationOptions); + } ); success = definiteResult.status === OperationStatus.Success; result = definiteResult; - await measureAsyncFn(`${PERF_PREFIX}:afterExecuteOperations`, () => - this.hooks.afterExecuteOperations.promise(definiteResult, executeOperationsContext) - ); - stopwatch.stop(); const message: string = `rush ${this.actionName} (${stopwatch.toString()})`; @@ -1013,145 +730,11 @@ export class PhasedScriptAction extends BaseScriptAction i } } - this._executionAbortController = undefined; - - if (invalidateOperation) { - const operationResults: ReadonlyMap | undefined = - result?.operationResults; - if (operationResults) { - for (const [operation, { status }] of operationResults) { - if (status === OperationStatus.Aborted) { - invalidateOperation(operation, 'aborted'); - } - } - } - } - if (!ignoreHooks) { measureFn(`${PERF_PREFIX}:doAfterTask`, () => this._doAfterTask()); } - if (this.parser.telemetry) { - const logEntry: ITelemetryData = measureFn(`${PERF_PREFIX}:prepareTelemetry`, () => { - const jsonOperationResults: Record = {}; - - const extraData: IPhasedCommandTelemetry = { - // Fields preserved across the command invocation - ...this._selectionParameters.getTelemetry(), - ...this.getParameterStringMap(), - isWatch, - // Fields specific to the current operation set - isInitial, - - countAll: 0, - countSuccess: 0, - countSuccessWithWarnings: 0, - countFailure: 0, - countBlocked: 0, - countFromCache: 0, - countSkipped: 0, - countNoOp: 0 - }; - - const { _changedProjectsOnlyParameter: changedProjectsOnlyParameter } = this; - if (changedProjectsOnlyParameter) { - // Overwrite this value since we allow changing it at runtime. - extraData[changedProjectsOnlyParameter.scopedLongName ?? changedProjectsOnlyParameter.longName] = - this._changedProjectsOnly; - } - - if (result) { - const { operationResults } = result; - - const nonSilentDependenciesByOperation: Map> = new Map(); - function getNonSilentDependencies(operation: Operation): ReadonlySet { - let realDependencies: Set | undefined = nonSilentDependenciesByOperation.get(operation); - if (!realDependencies) { - realDependencies = new Set(); - nonSilentDependenciesByOperation.set(operation, realDependencies); - for (const dependency of operation.dependencies) { - const dependencyRecord: IOperationExecutionResult | undefined = - operationResults.get(dependency); - if (dependencyRecord?.silent) { - for (const deepDependency of getNonSilentDependencies(dependency)) { - realDependencies.add(deepDependency); - } - } else { - realDependencies.add(dependency.name!); - } - } - } - return realDependencies; - } - - for (const [operation, operationResult] of operationResults) { - if (operationResult.silent) { - // Architectural operation. Ignore. - continue; - } - - const { _operationMetadataManager: operationMetadataManager } = - operationResult as OperationExecutionRecord; - - const { startTime, endTime } = operationResult.stopwatch; - jsonOperationResults[operation.name!] = { - startTimestampMs: startTime, - endTimestampMs: endTime, - nonCachedDurationMs: operationResult.nonCachedDurationMs, - wasExecutedOnThisMachine: operationMetadataManager?.wasCobuilt !== true, - result: operationResult.status, - dependencies: Array.from(getNonSilentDependencies(operation)).sort() - }; - - extraData.countAll++; - switch (operationResult.status) { - case OperationStatus.Success: - extraData.countSuccess++; - break; - case OperationStatus.SuccessWithWarning: - extraData.countSuccessWithWarnings++; - break; - case OperationStatus.Failure: - extraData.countFailure++; - break; - case OperationStatus.Blocked: - extraData.countBlocked++; - break; - case OperationStatus.FromCache: - extraData.countFromCache++; - break; - case OperationStatus.Skipped: - extraData.countSkipped++; - break; - case OperationStatus.NoOp: - extraData.countNoOp++; - break; - default: - // Do nothing. - break; - } - } - } - - const innerLogEntry: ITelemetryData = { - name: this.actionName, - durationInSeconds: stopwatch.duration, - result: success ? 'Succeeded' : 'Failed', - extraData, - operationResults: jsonOperationResults - }; - - return innerLogEntry; - }); - - measureFn(`${PERF_PREFIX}:beforeLog`, () => this.hooks.beforeLog.call(logEntry)); - - this.parser.telemetry.log(logEntry); - - this.parser.flushTelemetry(); - } - - if (!success && !isWatch) { + if (!success) { throw new AlreadyReportedError(); } } diff --git a/libraries/rush-lib/src/index.ts b/libraries/rush-lib/src/index.ts index a91cea23a98..abab202498f 100644 --- a/libraries/rush-lib/src/index.ts +++ b/libraries/rush-lib/src/index.ts @@ -143,12 +143,20 @@ export type { IRushConfigurationProjectForSnapshot } from './logic/incremental/InputsSnapshot'; -export type { IOperationRunner, IOperationRunnerContext } from './logic/operations/IOperationRunner'; export type { + IOperationRunner, + IOperationRunnerContext, + IOperationLastState +} from './logic/operations/IOperationRunner'; +export type { + IConfigurableOperation, + IBaseOperationExecutionResult, IExecutionResult, - IOperationExecutionResult + IOperationExecutionResult, + IOperationStateHashComponents } from './logic/operations/IOperationExecutionResult'; -export { type IOperationOptions, Operation } from './logic/operations/Operation'; +export { type IOperationOptions, type OperationEnabledState, Operation } from './logic/operations/Operation'; +export { type IParallelismScalar, type Parallelism } from './logic/operations/ParseParallelism'; export { OperationStatus } from './logic/operations/OperationStatus'; export type { ILogFilePaths } from './logic/operations/ProjectLogWritable'; @@ -168,9 +176,12 @@ export { export { type ICreateOperationsContext, - type IExecuteOperationsContext, + type IOperationGraphContext, + type IPhasedCommandPlugin, PhasedCommandHooks } from './pluginFramework/PhasedCommandHooks'; +export type { IOperationGraph, IOperationGraphIterationOptions } from './logic/operations/IOperationGraph'; +export { OperationGraphHooks } from './pluginFramework/OperationGraphHooks'; export type { IRushPlugin } from './pluginFramework/IRushPlugin'; export type { IBuiltInPluginConfiguration as _IBuiltInPluginConfiguration } from './pluginFramework/PluginLoader/BuiltInPluginLoader'; diff --git a/libraries/rush-lib/src/logic/ProjectWatcher.ts b/libraries/rush-lib/src/logic/ProjectWatcher.ts index 55d92b2d248..0e0c9adb877 100644 --- a/libraries/rush-lib/src/logic/ProjectWatcher.ts +++ b/libraries/rush-lib/src/logic/ProjectWatcher.ts @@ -7,23 +7,24 @@ import * as readline from 'node:readline'; import { once } from 'node:events'; import { getRepoRoot } from '@rushstack/package-deps-hash'; -import { AlreadyReportedError, Path, type FileSystemStats, FileSystem } from '@rushstack/node-core-library'; +import { Path } from '@rushstack/node-core-library'; import { Colorize, type ITerminal } from '@rushstack/terminal'; import { Git } from './Git'; -import type { IInputsSnapshot, GetInputsSnapshotAsyncFn } from './incremental/InputsSnapshot'; +import type { IInputsSnapshot } from './incremental/InputsSnapshot'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; -import { IS_WINDOWS } from '../utilities/executionUtilities'; +import type { IOperationGraph, IOperationGraphIterationOptions } from './operations/IOperationGraph'; +import type { Operation } from './operations/Operation'; +import { OperationStatus } from './operations/OperationStatus'; export interface IProjectWatcherOptions { - abortSignal: AbortSignal; - getInputsSnapshotAsync: GetInputsSnapshotAsyncFn; - debounceMs?: number; + graph: IOperationGraph; + debounceMs: number; rushConfiguration: RushConfiguration; - projectsToWatch: ReadonlySet; terminal: ITerminal; - initialSnapshot?: IInputsSnapshot | undefined; + /** Initial inputs snapshot; required so watcher can enumerate nested folders immediately */ + initialSnapshot: IInputsSnapshot; } export interface IProjectChangeResult { @@ -41,506 +42,436 @@ export interface IPromptGeneratorFunction { (isPaused: boolean): Iterable; } -interface IPathWatchOptions { - recurse: boolean; -} +const KEY_QUIT: 'q' = 'q'; +const KEY_ABORT: 'a' = 'a'; +const KEY_INVALIDATE: 'i' = 'i'; +const KEY_CLOSE_RUNNERS: 'x' = 'x'; +const KEY_DEBUG: 'd' = 'd'; +const KEY_VERBOSE: 'v' = 'v'; +const KEY_PAUSE_RESUME: 'w' = 'w'; +const KEY_BUILD: 'b' = 'b'; +const KEY_PARALLELISM_UP: '+' = '+'; +const KEY_PARALLELISM_DOWN: '-' = '-'; + +const KEYBIND_HELP: string = + `[${KEY_QUIT}]quit [${KEY_ABORT}]abort-iteration [${KEY_INVALIDATE}]invalidate ` + + `[${KEY_CLOSE_RUNNERS}]close-runners [${KEY_DEBUG}]debug [${KEY_VERBOSE}]verbose ` + + `[${KEY_PAUSE_RESUME}]pause/resume [${KEY_BUILD}]build [${KEY_PARALLELISM_UP}/${KEY_PARALLELISM_DOWN}]parallelism`; /** - * This class is for incrementally watching a set of projects in the repository for changes. - * - * We are manually using fs.watch() instead of `chokidar` because all we want from the file system watcher is a boolean - * signal indicating that "at least 1 file in a watched project changed". We then defer to getInputsSnapshotAsync (which - * is responsible for change detection in all incremental builds) to determine what actually chanaged. + * Watches a set of projects in the repository for file changes and triggers + * rebuild iterations on the operation graph. * - * Calling `waitForChange()` will return a promise that resolves when the package-deps of one or - * more projects differ from the value the previous time it was invoked. The first time will always resolve with the full selection. + * Uses `fs.watch()` rather than `chokidar` because only a boolean "something changed" + * signal is needed; actual change detection is deferred to `getInputsSnapshotAsync`. */ export class ProjectWatcher { - private readonly _abortSignal: AbortSignal; - private readonly _getInputsSnapshotAsync: GetInputsSnapshotAsyncFn; private readonly _debounceMs: number; - private readonly _repoRoot: string; private readonly _rushConfiguration: RushConfiguration; - private readonly _projectsToWatch: ReadonlySet; private readonly _terminal: ITerminal; + private readonly _graph: IOperationGraph; - private _initialSnapshot: IInputsSnapshot | undefined; - private _previousSnapshot: IInputsSnapshot | undefined; - private _forceChangedProjects: Map = new Map(); - private _resolveIfChanged: undefined | (() => Promise); - private _onAbort: undefined | (() => void); - private _getPromptLines: undefined | IPromptGeneratorFunction; - + private _repoRoot: string | undefined; + private _watchers: Map | undefined; + private _closePromises: Promise[] = []; + private _debounceHandle: NodeJS.Timeout | undefined; + private _isWatching: boolean = false; private _lastStatus: string | undefined; - private _renderedStatusLines: number; - - public isPaused: boolean = false; + private _renderedStatusLines: number = 0; + private _lastSnapshot: IInputsSnapshot | undefined; + private _stdinListening: boolean = false; + private _stdinHadRawMode: boolean | undefined; + private _onStdinDataBound: ((chunk: Buffer | string) => void) | undefined; public constructor(options: IProjectWatcherOptions) { - const { - abortSignal, - getInputsSnapshotAsync: snapshotProvider, - debounceMs = 1000, - rushConfiguration, - projectsToWatch, - terminal, - initialSnapshot: initialState - } = options; - - this._abortSignal = abortSignal; - abortSignal.addEventListener('abort', () => { - this._onAbort?.(); - }); + const { graph, debounceMs, rushConfiguration, terminal, initialSnapshot } = options; + this._graph = graph; this._debounceMs = debounceMs; this._rushConfiguration = rushConfiguration; - this._projectsToWatch = projectsToWatch; this._terminal = terminal; + this._lastSnapshot = initialSnapshot; // Seed snapshot const gitPath: string = new Git(rushConfiguration).getGitPathOrThrow(); this._repoRoot = Path.convertToSlashes(getRepoRoot(rushConfiguration.rushJsonFolder, gitPath)); - this._resolveIfChanged = undefined; - this._onAbort = undefined; - - this._initialSnapshot = initialState; - this._previousSnapshot = initialState; - - this._renderedStatusLines = 0; - this._getPromptLines = undefined; - this._getInputsSnapshotAsync = snapshotProvider; - } - - public pause(): void { - this.isPaused = true; - this._setStatus('Project watcher paused.'); - } - public resume(): void { - this.isPaused = false; - this._setStatus('Project watcher resuming...'); - if (this._resolveIfChanged) { - this._resolveIfChanged().catch(() => { - // Suppress unhandled promise rejection error - }); - } - } + // Initialize stdin listener early so keybinds are available immediately + this._ensureStdin(); - public invalidateProject(project: RushConfigurationProject, reason: string): boolean { - if (this._forceChangedProjects.has(project)) { - return false; - } + // Capture snapshot (if provided) prior to executing next iteration (will replace initial snapshot) + graph.hooks.beforeExecuteIterationAsync.tapPromise( + 'ProjectWatcher', + async ( + records: ReadonlyMap, + iterationOptions: IOperationGraphIterationOptions + ): Promise => { + this.clearStatus(); + this._lastSnapshot = iterationOptions.inputsSnapshot; + await this._stopWatchingAsync(); + } + ); - this._forceChangedProjects.set(project, reason); - return true; - } + // Start watching once execution loop enters waiting state + graph.hooks.onIdle.tap('ProjectWatcher', () => { + this._startWatching(); + }); - public invalidateAll(reason: string): void { - for (const project of this._projectsToWatch) { - this.invalidateProject(project, reason); - } + // Dispose stdin listener when session aborts + graph.abortController.signal.addEventListener( + 'abort', + () => { + this._disposeStdin(); + }, + { once: true } + ); } + /** + * Resets the rendered line count so the next status update does not attempt + * to overwrite previously rendered lines. + */ public clearStatus(): void { this._renderedStatusLines = 0; } + /** + * Re-renders the most recent status line (or a default) in place. + */ public rerenderStatus(): void { this._setStatus(this._lastStatus ?? 'Waiting for changes...'); } - public setPromptGenerator(promptGenerator: IPromptGeneratorFunction): void { - this._getPromptLines = promptGenerator; - } - /** - * Waits for a change to the package-deps of one or more of the selected projects, since the previous invocation. - * Will return immediately the first time it is invoked, since no state has been recorded. - * If no change is currently present, watches the source tree of all selected projects for file changes. - * `waitForChange` is not allowed to be called multiple times concurrently. + * Renders the given status message to the terminal, preceded by mode indicators + * and keybind help when stdin is active. Overwrites previously rendered status lines + * when not mid-execution. */ - public async waitForChangeAsync(onWatchingFiles?: () => void): Promise { - if (this.isPaused) { - this._setStatus(`Project watcher paused.`); - await new Promise((resolve) => { - this._resolveIfChanged = async () => resolve(); - }); + private _setStatus(status: string): void { + const graph: IOperationGraph = this._graph; + const isPaused: boolean = graph.pauseNextIteration === true; + const hasScheduledIteration: boolean = graph.hasScheduledIteration; + const modeLabel: string = isPaused ? 'PAUSED' : 'WATCHING'; + const pendingLabel: string = hasScheduledIteration ? ' PENDING' : ''; + const statusLines: string[] = [`[${modeLabel}${pendingLabel}] Watch Status: ${status}`]; + if (this._stdinListening) { + const lines: string[] = []; + // First line: modes + lines.push( + ` debug:${graph.debugMode ? 'on' : 'off'} verbose:${!graph.quietMode ? 'on' : 'off'} parallel:${graph.parallelism}` + ); + // Second line: keybind help kept concise to avoid overwhelming output + lines.push(` keys(active): ${KEYBIND_HELP}`); + statusLines.push(...lines.map((l) => ` ${l}`)); } - - const initialChangeResult: IProjectChangeResult = await this._computeChangedAsync(); - // Ensure that the new state is recorded so that we don't loop infinitely - this._commitChanges(initialChangeResult.inputsSnapshot); - if (initialChangeResult.changedProjects.size) { - // We can't call `clear()` here due to the async tick in the end of _computeChanged - for (const project of initialChangeResult.changedProjects) { - this._forceChangedProjects.delete(project); + if (graph.status !== OperationStatus.Executing) { + // If rendering during execution, don't try to clean previous output. + if (this._renderedStatusLines > 0) { + readline.cursorTo(process.stdout, 0); + readline.moveCursor(process.stdout, 0, -this._renderedStatusLines); + readline.clearScreenDown(process.stdout); } - // TODO: _forceChangedProjects might be non-empty here, which will result in an immediate rerun after the next - // run finishes. This is suboptimal, but the latency of _computeChanged is probably high enough that in practice - // all invalidations will have been picked up already. - return initialChangeResult; + this._renderedStatusLines = statusLines.length; } + this._lastStatus = status; + this._terminal.writeLine(Colorize.bold(Colorize.cyan(statusLines.join('\n')))); + } - const previousState: IInputsSnapshot = initialChangeResult.inputsSnapshot; - const repoRoot: string = Path.convertToSlashes(this._rushConfiguration.rushJsonFolder); - - // Map of path to whether config for the path - const pathsToWatch: Map = new Map(); - - // Node 12 supports the "recursive" parameter to fs.watch only on win32 and OSX - // https://nodejs.org/docs/latest-v12.x/api/fs.html#fs_caveats - const useNativeRecursiveWatch: boolean = IS_WINDOWS || os.platform() === 'darwin'; - - if (useNativeRecursiveWatch) { - // Watch the root non-recursively - pathsToWatch.set(repoRoot, { recurse: false }); - - // Watch the rush config folder non-recursively - pathsToWatch.set(Path.convertToSlashes(this._rushConfiguration.commonRushConfigFolder), { - recurse: false - }); - - for (const project of this._projectsToWatch) { - // Use recursive watch in individual project folders - pathsToWatch.set(Path.convertToSlashes(project.projectFolder), { recurse: true }); - } - } else { - for (const project of this._projectsToWatch) { - const projectState: ReadonlyMap = - previousState.getTrackedFileHashesForOperation(project); - - const prefixLength: number = project.projectFolder.length - repoRoot.length - 1; - // Watch files in the root of the project, or - for (const pathToWatch of ProjectWatcher._enumeratePathsToWatch(projectState.keys(), prefixLength)) { - pathsToWatch.set(`${this._repoRoot}/${pathToWatch}`, { recurse: true }); - } - } + /** + * Begins watching the file system for changes in all tracked project folders. + * On platforms without native recursive watch support (Linux), enumerates nested + * folders from the last snapshot to set up individual watchers. + */ + private _startWatching(): void { + if (this._isWatching) { + return; } + this._isWatching = true; + const sessionAbortSignal: AbortSignal = this._graph.abortController.signal; + const repoRoot: string = Path.convertToSlashes(this._rushConfiguration.rushJsonFolder); + const useNativeRecursiveWatch: boolean = os.platform() === 'win32' || os.platform() === 'darwin'; + const operations: ReadonlySet = this._graph.operations; - if (this._abortSignal.aborted) { - return initialChangeResult; + const projectFolders: Set = new Set(); + for (const op of operations) { + projectFolders.add(Path.convertToSlashes(op.associatedProject.projectFolder)); } - const watchers: Map = new Map(); - const closePromises: Promise[] = []; - - const watchedResult: IProjectChangeResult = await new Promise( - (resolve: (result: IProjectChangeResult) => void, reject: (err: Error) => void) => { - let timeout: NodeJS.Timeout | undefined; - let terminated: boolean = false; - - const terminal: ITerminal = this._terminal; - - const debounceMs: number = this._debounceMs; - - const abortSignal: AbortSignal = this._abortSignal; - - this.clearStatus(); - - this._onAbort = function onAbort(): void { - if (timeout) { - clearTimeout(timeout); - } - terminated = true; - resolve(initialChangeResult); - }; - - if (abortSignal.aborted) { - return this._onAbort(); + // Derive nested folder list if on Linux (no native recursive) and snapshot available + let foldersToWatch: Set = new Set(); + if (!useNativeRecursiveWatch && this._lastSnapshot) { + for (const op of operations) { + const { associatedProject: rushProject } = op; + const tracked: ReadonlyMap | undefined = + this._lastSnapshot.getTrackedFileHashesForOperation(rushProject); + if (!tracked) { + continue; } - - const resolveIfChanged: () => Promise = (this._resolveIfChanged = async (): Promise => { - timeout = undefined; - if (terminated) { - return; - } - - try { - if (this.isPaused) { - this._setStatus(`Project watcher paused.`); - return; - } - - this._setStatus(`Evaluating changes to tracked files...`); - const result: IProjectChangeResult = await this._computeChangedAsync(); - this._setStatus(`Finished analyzing.`); - - // Need an async tick to allow for more file system events to be handled - process.nextTick(() => { - if (timeout) { - // If another file has changed, wait for another pass. - this._setStatus(`More file changes detected, aborting.`); - return; - } - - // Since there are multiple async ticks since the projects were enumerated in _computeChanged, - // more could have been added in the interaval. Check and debounce. - for (const project of this._forceChangedProjects.keys()) { - if (!result.changedProjects.has(project)) { - this._setStatus(`More invalidations occurred, aborting.`); - timeout = setTimeout(resolveIfChanged, debounceMs); - return; - } - } - - this._commitChanges(result.inputsSnapshot); - - const hasForcedChanges: boolean = this._forceChangedProjects.size > 0; - if (hasForcedChanges) { - this._setStatus( - `Projects were invalidated: ${Array.from(new Set(this._forceChangedProjects.values())).join( - ', ' - )}` - ); - this.clearStatus(); - } - this._forceChangedProjects.clear(); - - if (result.changedProjects.size) { - terminated = true; - terminal.writeLine(); - resolve(result); - } else { - this._setStatus(`No changes detected to tracked files.`); - } - }); - } catch (err) { - // eslint-disable-next-line require-atomic-updates - terminated = true; - terminal.writeLine(); - reject(err as NodeJS.ErrnoException); - } - }); - - for (const [pathToWatch, { recurse }] of pathsToWatch) { - addWatcher(pathToWatch, recurse); - } - - if (onWatchingFiles) { - onWatchingFiles(); - } - - this._setStatus(`Waiting for changes...`); - - function onError(err: Error): void { - if (terminated) { - return; - } - - terminated = true; - terminal.writeLine(); - reject(err); - } - - function addWatcher(watchedPath: string, recursive: boolean): void { - if (watchers.has(watchedPath)) { - return; - } - const listener: fs.WatchListener = changeListener(watchedPath, recursive); - const watcher: fs.FSWatcher = fs.watch( - watchedPath, - { - encoding: 'utf-8', - recursive: recursive && useNativeRecursiveWatch, - signal: abortSignal - }, - listener - ); - watchers.set(watchedPath, watcher); - watcher.once('error', (err) => { - watcher.close(); - onError(err); - }); - closePromises.push( - once(watcher, 'close').then(() => { - watchers.delete(watchedPath); - watcher.removeAllListeners(); - watcher.unref(); - }) - ); - } - - function innerListener( - root: string, - recursive: boolean, - event: string, - fileName: string | null - ): void { - try { - if (terminated) { - return; - } - - if (fileName === '.git' || fileName === 'node_modules') { - return; - } - - // Handling for added directories - if (recursive && !useNativeRecursiveWatch) { - const decodedName: string = fileName ? fileName.toString() : ''; - const normalizedName: string = decodedName && Path.convertToSlashes(decodedName); - const fullName: string = normalizedName && `${root}/${normalizedName}`; - - if (fullName && !watchers.has(fullName)) { - try { - const stat: FileSystemStats = FileSystem.getStatistics(fullName); - if (stat.isDirectory()) { - addWatcher(fullName, true); - } - } catch (err) { - const code: string | undefined = (err as NodeJS.ErrnoException).code; - - if (code !== 'ENOENT' && code !== 'ENOTDIR') { - throw err; - } - } - } - } - - // Use a timeout to debounce changes, e.g. bulk copying files into the directory while the watcher is running. - if (timeout) { - clearTimeout(timeout); - } - - timeout = setTimeout(resolveIfChanged, debounceMs); - } catch (err) { - terminated = true; - terminal.writeLine(); - reject(err as NodeJS.ErrnoException); - } - } - - function changeListener(root: string, recursive: boolean): fs.WatchListener { - return innerListener.bind(0, root, recursive); + const prefixLength: number = rushProject.projectFolder.length - repoRoot.length - 1; + for (const relPrefix of _enumeratePathsToWatch(tracked.keys(), prefixLength)) { + foldersToWatch.add(`${this._repoRoot}/${relPrefix}`); } } - ).finally(() => { - this._onAbort = undefined; - this._resolveIfChanged = undefined; - }); - - this._terminal.writeDebugLine(`Closing watchers...`); - - for (const watcher of watchers.values()) { - watcher.close(); + } + if (!useNativeRecursiveWatch && foldersToWatch.size === 0) { + // Fallback to project roots if snapshot missing + foldersToWatch = projectFolders; } - await Promise.all(closePromises); - this._terminal.writeDebugLine(`Closed ${closePromises.length} watchers`); + const watchers: Map = (this._watchers = new Map()); - return watchedResult; - } + const addWatcher = (watchedPath: string, recursive: boolean): void => { + if (watchers.has(watchedPath)) { + return; + } + try { + const watcher: fs.FSWatcher = fs.watch( + watchedPath, + { + encoding: 'utf-8', + recursive: recursive && useNativeRecursiveWatch, + signal: sessionAbortSignal + }, + (eventType, fileName) => this._onFsEvent(fileName) + ); + watchers.set(watchedPath, watcher); + this._closePromises.push( + once(watcher, 'close').then(() => { + watchers.delete(watchedPath); + watcher.removeAllListeners(); + watcher.unref(); + }) + ); + } catch (e) { + this._terminal.writeDebugLine(`Failed to watch path ${watchedPath}: ${(e as Error).message}`); + } + }; - private _setStatus(status: string): void { - const statusLines: string[] = [ - `[${this.isPaused ? 'PAUSED' : 'WATCHING'}] Watch Status: ${status}`, - ...(this._getPromptLines?.(this.isPaused) ?? []) - ]; - - if (this._renderedStatusLines > 0) { - readline.cursorTo(process.stdout, 0); - readline.moveCursor(process.stdout, 0, -this._renderedStatusLines); - readline.clearScreenDown(process.stdout); + // Always watch repo root and common config + addWatcher(repoRoot, false); + addWatcher(Path.convertToSlashes(this._rushConfiguration.commonRushConfigFolder), false); + if (useNativeRecursiveWatch) { + for (const folder of projectFolders) { + addWatcher(folder, true); + } + } else { + for (const folder of foldersToWatch) { + addWatcher(folder, true); + } } - this._renderedStatusLines = statusLines.length; - this._lastStatus = status; - - this._terminal.writeLine(Colorize.bold(Colorize.cyan(statusLines.join('\n')))); + this._setStatus('Waiting for changes...'); } /** - * Determines which, if any, projects (within the selection) have new hashes for files that are not in .gitignore + * Closes all active file system watchers and waits for their close events to settle. */ - private async _computeChangedAsync(): Promise { - const currentSnapshot: IInputsSnapshot | undefined = await this._getInputsSnapshotAsync(); - - if (!currentSnapshot) { - throw new AlreadyReportedError(); + private async _stopWatchingAsync(): Promise { + if (!this._isWatching) { + return; } - - const previousSnapshot: IInputsSnapshot | undefined = this._previousSnapshot; - - if (!previousSnapshot) { - return { - changedProjects: this._projectsToWatch, - inputsSnapshot: currentSnapshot - }; + this._isWatching = false; + if (this._debounceHandle) { + clearTimeout(this._debounceHandle); + this._debounceHandle = undefined; } - - const changedProjects: Set = new Set(); - for (const project of this._projectsToWatch) { - const previous: ReadonlyMap | undefined = - previousSnapshot.getTrackedFileHashesForOperation(project); - const current: ReadonlyMap | undefined = - currentSnapshot.getTrackedFileHashesForOperation(project); - - if (ProjectWatcher._haveProjectDepsChanged(previous, current)) { - // May need to detect if the nature of the change will break the process, e.g. changes to package.json - changedProjects.add(project); + if (this._watchers) { + for (const watcher of this._watchers.values()) { + watcher.close(); } } + await Promise.all(this._closePromises); + this._closePromises = []; + this._watchers = undefined; + this._terminal.writeDebugLine('ProjectWatcher: watchers stopped'); + } - for (const project of this._forceChangedProjects.keys()) { - changedProjects.add(project); + /** + * Handles a raw file system event by debouncing and scheduling an iteration. + * Ignores changes to `.git` and `node_modules`. + */ + private _onFsEvent(fileName: string | null): void { + if (fileName === '.git' || fileName === 'node_modules') { + return; } + if (this._debounceHandle) { + clearTimeout(this._debounceHandle); + } + this._debounceHandle = setTimeout(() => this._scheduleIteration(), this._debounceMs); + } - return { - changedProjects, - inputsSnapshot: currentSnapshot - }; + /** + * Schedules a new execution iteration on the graph in response to detected file changes. + */ + private _scheduleIteration(): void { + this._setStatus('File change detected. Queuing new iteration...'); + this._graph + .scheduleIterationAsync({}) + .catch((e: unknown) => + this._terminal.writeErrorLine(`Failed to queue iteration: ${(e as Error).message}`) + ); } - private _commitChanges(state: IInputsSnapshot): void { - this._previousSnapshot = state; - if (!this._initialSnapshot) { - this._initialSnapshot = state; + /** + * Sets up a raw-mode stdin listener so the user can interact with the watch session + * via single-key keybinds. Captures the previous raw-mode state for restoration on dispose. + */ + private _ensureStdin(): void { + if (this._stdinListening || !process.stdin.isTTY) { + return; + } + const stdin: NodeJS.ReadStream = process.stdin as NodeJS.ReadStream; + // Node's ReadStream has an undocumented isRaw property when setRawMode has been used. + // Capture it in a type-safe way. + this._stdinHadRawMode = + typeof (stdin as unknown as { isRaw?: boolean }).isRaw === 'boolean' + ? (stdin as unknown as { isRaw?: boolean }).isRaw + : undefined; // capture existing raw state + try { + stdin.setRawMode?.(true); + } catch { + // ignore if cannot set raw mode } + stdin.resume(); + stdin.setEncoding('utf8'); + const handler = (chunk: Buffer | string): void => this._onStdinData(chunk.toString()); + stdin.on('data', handler); + this._onStdinDataBound = handler; + this._stdinListening = true; } /** - * Tests for inequality of the passed Maps. Order invariant. - * - * @returns `true` if the maps are different, `false` otherwise + * Removes the stdin listener and restores the previous raw-mode state. */ - private static _haveProjectDepsChanged( - prev: ReadonlyMap | undefined, - next: ReadonlyMap | undefined - ): boolean { - if (!prev && !next) { - return false; + private _disposeStdin(): void { + if (!this._stdinListening) { + return; } - - if (!prev || !next) { - return true; + const stdin: NodeJS.ReadStream = process.stdin as NodeJS.ReadStream; + if (this._onStdinDataBound) { + stdin.off('data', this._onStdinDataBound); + this._onStdinDataBound = undefined; } - - if (prev.size !== next.size) { - return true; + try { + stdin.setRawMode?.(!!this._stdinHadRawMode); + } catch { + // ignore } + stdin.unref(); + this._stdinListening = false; + } - for (const [key, value] of prev) { - if (next.get(key) !== value) { - return true; + /** + * Processes a chunk of stdin data, dispatching each character to the appropriate + * keybind action on the operation graph. + */ + private _onStdinData(chunk: string): void { + const graph: IOperationGraph = this._graph; + if (!chunk) return; + for (const ch of chunk) { + // Once aborted, only respond to Ctrl+C (force exit) + if (graph.abortController.signal.aborted) { + if (ch === '\u0003') { + process.exit(1); + } + continue; + } + switch (ch) { + case '\u0003': + case KEY_QUIT: { + this._terminal.writeLine('Aborting watch session... (Ctrl+C to force exit)'); + graph.abortController.abort(); + break; + } + case KEY_ABORT: { + void graph.abortCurrentIterationAsync().then(() => { + this._setStatus('Current iteration aborted'); + }); + break; + } + case KEY_INVALIDATE: { + graph.invalidateOperations(undefined, 'manual-invalidation'); + this._setStatus('All operations invalidated'); + break; + } + case KEY_CLOSE_RUNNERS: { + void graph.closeRunnersAsync().then(() => { + this._setStatus('Closed all runners'); + }); + break; + } + case KEY_DEBUG: { + graph.debugMode = !graph.debugMode; + this._setStatus(`Debug mode ${graph.debugMode ? 'enabled' : 'disabled'}`); + break; + } + case KEY_VERBOSE: { + graph.quietMode = !graph.quietMode; + this._setStatus(`Verbose mode ${!graph.quietMode ? 'enabled' : 'disabled'}`); + break; + } + case KEY_PAUSE_RESUME: { + graph.pauseNextIteration = !graph.pauseNextIteration; + this._setStatus(graph.pauseNextIteration ? 'Watch paused' : 'Watch resumed'); + break; + } + case KEY_PARALLELISM_UP: + case '=': { + this._adjustParallelism(1); + break; + } + case KEY_PARALLELISM_DOWN: { + this._adjustParallelism(-1); + break; + } + case KEY_BUILD: { + void graph.scheduleIterationAsync({ startTime: performance.now() }).then((queued) => { + if (queued) { + if (graph.pauseNextIteration === true) { + void graph.executeScheduledIterationAsync(); + } + this._setStatus('Build iteration queued'); + } else { + this._setStatus('No work to queue'); + } + }); + break; + } + default: { + // ignore other keys + break; + } } } - - return false; } - private static *_enumeratePathsToWatch(paths: Iterable, prefixLength: number): Iterable { - for (const path of paths) { - const rootSlashIndex: number = path.indexOf('/', prefixLength); - - if (rootSlashIndex < 0) { - yield path; - return; - } - - yield path.slice(0, rootSlashIndex); + /** + * Adjusts the parallelism on the operation graph by the given delta + * and reports the result. + */ + private _adjustParallelism(delta: number): void { + const graph: IOperationGraph = this._graph; + const previous: number = graph.parallelism; + graph.parallelism = previous + delta; // setter will clamp/normalize + const effective: number = graph.parallelism; + this._setStatus(`Parallelism ${effective !== previous ? 'set to' : 'remains'} ${effective}`); + } +} - let slashIndex: number = path.indexOf('/', rootSlashIndex + 1); - while (slashIndex >= 0) { - yield path.slice(0, slashIndex); - slashIndex = path.indexOf('/', slashIndex + 1); - } +/** + * Given an iterable of repo-relative file paths, yields the set of directory prefixes + * that should be watched to cover those files. Used on platforms without native recursive + * watch support to enumerate nested folders. + */ +function* _enumeratePathsToWatch(paths: Iterable, prefixLength: number): Iterable { + for (const path of paths) { + const rootSlashIndex: number = path.indexOf('/', prefixLength); + if (rootSlashIndex < 0) { + yield path; + return; + } + yield path.slice(0, rootSlashIndex); + let slashIndex: number = path.indexOf('/', rootSlashIndex + 1); + while (slashIndex >= 0) { + yield path.slice(0, slashIndex); + slashIndex = path.indexOf('/', slashIndex + 1); } } } diff --git a/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts b/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts index 0abf76c221b..58ed4b8cb41 100644 --- a/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts +++ b/libraries/rush-lib/src/logic/buildCache/OperationBuildCache.ts @@ -13,7 +13,7 @@ import type { ICloudBuildCacheProvider } from './ICloudBuildCacheProvider'; import type { FileSystemBuildCacheProvider } from './FileSystemBuildCacheProvider'; import { TarExecutable } from '../../utilities/TarExecutable'; import { EnvironmentVariableNames } from '../../api/EnvironmentConfiguration'; -import type { IOperationExecutionResult } from '../operations/IOperationExecutionResult'; +import type { IBaseOperationExecutionResult } from '../operations/IOperationExecutionResult'; /** * @internal @@ -116,7 +116,7 @@ export class OperationBuildCache { } public static forOperation( - executionResult: IOperationExecutionResult, + executionResult: IBaseOperationExecutionResult, options: IOperationBuildCacheOptions ): OperationBuildCache { const { buildCacheConfiguration, terminal, excludeAppleDoubleFiles } = options; diff --git a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts index 85376112da5..dce4ed3785a 100644 --- a/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/BuildPlanPlugin.ts @@ -4,14 +4,15 @@ import type { ITerminal } from '@rushstack/terminal'; import type { - IExecuteOperationsContext, + IOperationGraphContext, IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; +import type { IOperationGraph, IOperationGraphIterationOptions } from './IOperationGraph'; import type { Operation } from './Operation'; import { clusterOperations, type IOperationBuildCacheContext } from './CacheableOperationPlugin'; import { DisjointSet } from '../cobuild/DisjointSet'; -import type { IOperationExecutionResult } from './IOperationExecutionResult'; +import type { IConfigurableOperation } from './IOperationExecutionResult'; import { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; const PLUGIN_NAME: 'BuildPlanPlugin' = 'BuildPlanPlugin'; @@ -41,13 +42,20 @@ export class BuildPlanPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { const terminal: ITerminal = this._terminal; - hooks.beforeExecuteOperations.tap(PLUGIN_NAME, createBuildPlan); + + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph: IOperationGraph, context: IOperationGraphContext) => { + graph.hooks.configureIteration.tap(PLUGIN_NAME, (currentStates, lastStates, iterationOptions) => { + createBuildPlan(currentStates, iterationOptions, context); + }); + }); function createBuildPlan( - recordByOperation: Map, - context: IExecuteOperationsContext + recordByOperation: ReadonlyMap, + iterationOptions: IOperationGraphIterationOptions, + context: IOperationGraphContext ): void { - const { projectConfigurations, inputsSnapshot } = context; + const { inputsSnapshot } = iterationOptions; + const { projectConfigurations } = context; const disjointSet: DisjointSet = new DisjointSet(); const operations: Operation[] = [...recordByOperation.keys()]; for (const operation of operations) { diff --git a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts index 90af287b9dd..46bf38715bb 100644 --- a/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts @@ -33,10 +33,11 @@ import type { Operation } from './Operation'; import type { IOperationRunnerContext } from './IOperationRunner'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { - IExecuteOperationsContext, + IOperationGraphContext, IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; +import type { IOperationGraph, IOperationGraphIterationOptions } from './IOperationGraph'; import type { BuildCacheConfiguration } from '../../api/BuildCacheConfiguration'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; import type { OperationExecutionRecord } from './OperationExecutionRecord'; @@ -111,458 +112,467 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin { excludeAppleDoubleFiles } = this._options; - hooks.beforeExecuteOperations.tap( - PLUGIN_NAME, - ( - recordByOperation: Map, - context: IExecuteOperationsContext - ): void => { - const { isIncrementalBuildAllowed, inputsSnapshot, projectConfigurations, isInitial } = context; - - if (!inputsSnapshot) { - throw new Error( - `Build cache is only supported if running in a Git repository. Either disable the build cache or run Rush in a Git repository.` - ); - } - - const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildFeatureEnabled - ? new DisjointSet() - : undefined; - - for (const [operation, record] of recordByOperation) { - const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation; - if (!runner) { - return; + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph: IOperationGraph, context: IOperationGraphContext) => { + graph.hooks.beforeExecuteIterationAsync.tap( + PLUGIN_NAME, + ( + recordByOperation: ReadonlyMap, + iterationOptions: IOperationGraphIterationOptions + ): undefined => { + const { inputsSnapshot } = iterationOptions; + + if (!inputsSnapshot) { + throw new Error( + `Build cache is only supported if running in a Git repository. Either disable the build cache or run Rush in a Git repository.` + ); } - const { name: phaseName } = associatedPhase; + const { isIncrementalBuildAllowed, projectConfigurations } = context; + const { cacheWriteEnabled } = buildCacheConfiguration; - const projectConfiguration: RushProjectConfiguration | undefined = - projectConfigurations.get(associatedProject); + const disjointSet: DisjointSet | undefined = cobuildConfiguration?.cobuildFeatureEnabled + ? new DisjointSet() + : undefined; - // This value can *currently* be cached per-project, but in the future the list of files will vary - // depending on the selected phase. - const fileHashes: ReadonlyMap | undefined = - inputsSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName); + for (const [operation, record] of recordByOperation) { + const { associatedProject, associatedPhase, runner, settings: operationSettings } = operation; + if (!runner) { + return; + } - const cacheDisabledReason: string | undefined = projectConfiguration - ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp) - : `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + - 'or one provided by a rig, so it does not support caching.'; + const { name: phaseName } = associatedPhase; - const metadataFolderPath: string | undefined = record.metadataFolderPath; + const projectConfiguration: RushProjectConfiguration | undefined = + projectConfigurations.get(associatedProject); - const outputFolderNames: string[] = metadataFolderPath ? [metadataFolderPath] : []; - const configuredOutputFolderNames: string[] | undefined = operationSettings?.outputFolderNames; - if (configuredOutputFolderNames) { - for (const folderName of configuredOutputFolderNames) { - outputFolderNames.push(folderName); - } - } + // This value can *currently* be cached per-project, but in the future the list of files will vary + // depending on the selected phase. + const fileHashes: ReadonlyMap | undefined = + inputsSnapshot.getTrackedFileHashesForOperation(associatedProject, phaseName); - disjointSet?.add(operation); - - const buildCacheContext: IOperationBuildCacheContext = { - // Supports cache writes by default for initial operations. - // Don't write during watch runs for performance reasons (and to avoid flooding the cache) - isCacheWriteAllowed: isInitial, - isCacheReadAllowed: isIncrementalBuildAllowed, - operationBuildCache: undefined, - outputFolderNames, - cacheDisabledReason, - cobuildLock: undefined, - cobuildClusterId: undefined, - buildCacheTerminal: undefined, - buildCacheTerminalWritable: undefined, - periodicCallback: new PeriodicCallback({ - interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 - }), - cacheRestored: false, - isCacheReadAttempted: false - }; - // Upstream runners may mutate the property of build cache context for downstream runners - this._buildCacheContextByOperation.set(operation, buildCacheContext); - } - - if (disjointSet) { - clusterOperations(disjointSet, this._buildCacheContextByOperation); - for (const operationSet of disjointSet.getAllSets()) { - if (cobuildConfiguration?.cobuildFeatureEnabled && cobuildConfiguration.cobuildContextId) { - // Get a deterministic ordered array of operations, which is important to get a deterministic cluster id. - const groupedOperations: Operation[] = Array.from(operationSet); - Sort.sortBy(groupedOperations, (operation: Operation) => { - return operation.name; - }); + const cacheDisabledReason: string | undefined = projectConfiguration + ? projectConfiguration.getCacheDisabledReason(fileHashes.keys(), phaseName, operation.isNoOp) + : `Project does not have a ${RushConstants.rushProjectConfigFilename} configuration file, ` + + 'or one provided by a rig, so it does not support caching.'; - // Generates cluster id, cluster id comes from the project folder and operation name of all operations in the same cluster. - const hash: crypto.Hash = crypto.createHash('sha1'); - for (const operation of groupedOperations) { - const { associatedPhase: phase, associatedProject: project } = operation; - hash.update(project.projectRelativeFolder); - hash.update(RushConstants.hashDelimiter); - hash.update(operation.name ?? phase.name); - hash.update(RushConstants.hashDelimiter); + const outputFolderNames: string[] = [record.metadataFolderPath]; + const configuredOutputFolderNames: string[] | undefined = operationSettings?.outputFolderNames; + if (configuredOutputFolderNames) { + for (const folderName of configuredOutputFolderNames) { + outputFolderNames.push(folderName); } - const cobuildClusterId: string = hash.digest('hex'); + } - // Assign same cluster id to all operations in the same cluster. - for (const record of groupedOperations) { - const buildCacheContext: IOperationBuildCacheContext = - this._getBuildCacheContextByOperationOrThrow(record); - buildCacheContext.cobuildClusterId = cobuildClusterId; + disjointSet?.add(operation); + + const buildCacheContext: IOperationBuildCacheContext = { + // Supports cache writes by default for initial operations. + // Don't write during watch runs for performance reasons (and to avoid flooding the cache) + isCacheWriteAllowed: cacheWriteEnabled, + isCacheReadAllowed: isIncrementalBuildAllowed, + operationBuildCache: undefined, + outputFolderNames, + cacheDisabledReason, + cobuildLock: undefined, + cobuildClusterId: undefined, + buildCacheTerminal: undefined, + buildCacheTerminalWritable: undefined, + periodicCallback: new PeriodicCallback({ + interval: PERIODIC_CALLBACK_INTERVAL_IN_SECONDS * 1000 + }), + cacheRestored: false, + isCacheReadAttempted: false + }; + // Upstream runners may mutate the property of build cache context for downstream runners + this._buildCacheContextByOperation.set(operation, buildCacheContext); + } + + if (disjointSet) { + clusterOperations(disjointSet, this._buildCacheContextByOperation); + for (const operationSet of disjointSet.getAllSets()) { + if (cobuildConfiguration?.cobuildFeatureEnabled && cobuildConfiguration.cobuildContextId) { + // Get a deterministic ordered array of operations, which is important to get a deterministic cluster id. + const groupedOperations: Operation[] = Array.from(operationSet); + Sort.sortBy(groupedOperations, (operation: Operation) => { + return operation.name; + }); + + // Generates cluster id, cluster id comes from the project folder and operation name of all operations in the same cluster. + const hash: crypto.Hash = crypto.createHash('sha1'); + for (const operation of groupedOperations) { + const { associatedPhase: phase, associatedProject: project } = operation; + hash.update(project.projectRelativeFolder); + hash.update(RushConstants.hashDelimiter); + hash.update(operation.name ?? phase.name); + hash.update(RushConstants.hashDelimiter); + } + const cobuildClusterId: string = hash.digest('hex'); + + // Assign same cluster id to all operations in the same cluster. + for (const record of groupedOperations) { + const buildCacheContext: IOperationBuildCacheContext = + this._getBuildCacheContextByOperationOrThrow(record); + buildCacheContext.cobuildClusterId = cobuildClusterId; + } } } } } - } - ); - - hooks.beforeExecuteOperation.tapPromise( - PLUGIN_NAME, - async ( - runnerContext: IOperationRunnerContext & IOperationExecutionResult - ): Promise => { - if (this._buildCacheContextByOperation.size === 0) { - return; - } + ); - const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByOperation(runnerContext.operation); + graph.hooks.beforeExecuteOperationAsync.tapPromise( + PLUGIN_NAME, + async ( + runnerContext: IOperationRunnerContext & IOperationExecutionResult + ): Promise => { + if (this._buildCacheContextByOperation.size === 0) { + return; + } - if (!buildCacheContext) { - return; - } + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByOperation(runnerContext.operation); - const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; + if (!buildCacheContext) { + return; + } - const { - associatedProject: project, - associatedPhase: phase, - runner, - _operationMetadataManager: operationMetadataManager, - operation - } = record; + const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; - if (!operation.enabled || !runner?.cacheable) { - return; - } + const { + associatedProject: project, + associatedPhase: phase, + runner, + _operationMetadataManager: operationMetadataManager, + operation + } = record; - const runBeforeExecute = async (): Promise => { - if ( - !buildCacheContext.buildCacheTerminal || - buildCacheContext.buildCacheTerminalWritable?.isOpen === false - ) { - // The writable does not exist or has been closed, re-create one - // eslint-disable-next-line require-atomic-updates - buildCacheContext.buildCacheTerminal = await this._createBuildCacheTerminalAsync({ - record, - buildCacheContext, - buildCacheEnabled: buildCacheConfiguration?.buildCacheEnabled, - rushProject: project, - logFilenameIdentifier: operation.logFilenameIdentifier, - quietMode: record.quietMode, - debugMode: record.debugMode - }); + if (!record.enabled || !runner?.cacheable) { + return; } - const buildCacheTerminal: ITerminal = buildCacheContext.buildCacheTerminal; - - let operationBuildCache: OperationBuildCache | undefined = this._tryGetOperationBuildCache({ - buildCacheContext, - buildCacheConfiguration, - terminal: buildCacheTerminal, - record, - excludeAppleDoubleFiles - }); - - // Try to acquire the cobuild lock - let cobuildLock: CobuildLock | undefined; - if (cobuildConfiguration?.cobuildFeatureEnabled) { + const runBeforeExecute = async (): Promise => { if ( - cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && - operation.consumers.size === 0 && - !operationBuildCache + !buildCacheContext.buildCacheTerminal || + buildCacheContext.buildCacheTerminalWritable?.isOpen === false ) { - // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get - // a log files only project build cache - operationBuildCache = await this._tryGetLogOnlyOperationBuildCacheAsync({ - buildCacheConfiguration, - cobuildConfiguration, - buildCacheContext, + // The writable does not exist or has been closed, re-create one + // eslint-disable-next-line require-atomic-updates + buildCacheContext.buildCacheTerminal = await this._createBuildCacheTerminalAsync({ record, - terminal: buildCacheTerminal, - excludeAppleDoubleFiles + buildCacheContext, + buildCacheEnabled: buildCacheConfiguration?.buildCacheEnabled, + rushProject: project, + logFilenameIdentifier: operation.logFilenameIdentifier, + quietMode: record.quietMode, + debugMode: record.debugMode }); - if (operationBuildCache) { - buildCacheTerminal.writeVerboseLine( - `Log files only build cache is enabled for the project "${project.packageName}" because the cobuild leaf project log only is allowed` - ); - } else { - buildCacheTerminal.writeWarningLine( - `Failed to get log files only build cache for the project "${project.packageName}"` - ); - } } - cobuildLock = await this._tryGetCobuildLockAsync({ + const buildCacheTerminal: ITerminal = buildCacheContext.buildCacheTerminal; + + let operationBuildCache: OperationBuildCache | undefined = this._tryGetOperationBuildCache({ buildCacheContext, - operationBuildCache, - cobuildConfiguration, - packageName: project.packageName, - phaseName: phase.name + buildCacheConfiguration, + terminal: buildCacheTerminal, + record, + excludeAppleDoubleFiles }); - } - // eslint-disable-next-line require-atomic-updates -- we are mutating the build cache context intentionally - buildCacheContext.cobuildLock = cobuildLock; - - // If possible, we want to skip this operation -- either by restoring it from the - // cache, if caching is enabled, or determining that the project - // is unchanged (using the older incremental execution logic). These two approaches, - // "caching" and "skipping", are incompatible, so only one applies. - // - // Note that "caching" and "skipping" take two different approaches - // to tracking dependents: - // - // - For caching, "isCacheReadAllowed" is set if a project supports - // incremental builds, and determining whether this project or a dependent - // has changed happens inside the hashing logic. - // - - const { error: errorLogPath } = getProjectLogFilePaths({ - project, - logFilenameIdentifier: operation.logFilenameIdentifier - }); - const restoreCacheAsync = async ( - // TODO: Investigate if `operationBuildCacheForRestore` is always the same instance as `operationBuildCache` - // above, and if it is, remove this parameter - operationBuildCacheForRestore: OperationBuildCache | undefined, - specifiedCacheId?: string - ): Promise => { - buildCacheContext.isCacheReadAttempted = true; - const restoreFromCacheSuccess: boolean | undefined = - await operationBuildCacheForRestore?.tryRestoreFromCacheAsync( - buildCacheTerminal, - specifiedCacheId - ); - if (restoreFromCacheSuccess) { - buildCacheContext.cacheRestored = true; - await runnerContext.runWithTerminalAsync( - async (taskTerminal, terminalProvider) => { - // Restore the original state of the operation without cache - await operationMetadataManager?.tryRestoreAsync({ - terminalProvider, - terminal: buildCacheTerminal, - errorLogPath, - cobuildContextId: cobuildConfiguration?.cobuildContextId, - cobuildRunnerId: cobuildConfiguration?.cobuildRunnerId - }); - }, - { createLogFile: false } - ); - } - return !!restoreFromCacheSuccess; - }; - if (cobuildLock) { - // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to - // the build cache once completed, but does not retrieve them (since the "incremental" - // flag is disabled). However, we still need a cobuild to be able to retrieve a finished - // build from another cobuild in this case. - const cobuildCompletedState: ICobuildCompletedState | undefined = - await cobuildLock.getCompletedStateAsync(); - if (cobuildCompletedState) { - const { status, cacheId } = cobuildCompletedState; - - if (record.operation.settings?.allowCobuildWithoutCache) { - // This should only be enabled if the experiment for cobuild orchestration is enabled. - return status; + // Try to acquire the cobuild lock + let cobuildLock: CobuildLock | undefined; + if (cobuildConfiguration?.cobuildFeatureEnabled) { + if ( + cobuildConfiguration?.cobuildLeafProjectLogOnlyAllowed && + operation.consumers.size === 0 && + !operationBuildCache + ) { + // When the leaf project log only is allowed and the leaf project is build cache "disabled", try to get + // a log files only project build cache + operationBuildCache = await this._tryGetLogOnlyOperationBuildCacheAsync({ + buildCacheConfiguration, + cobuildConfiguration, + buildCacheContext, + record, + terminal: buildCacheTerminal, + excludeAppleDoubleFiles + }); + if (operationBuildCache) { + buildCacheTerminal.writeVerboseLine( + `Log files only build cache is enabled for the project "${project.packageName}" because the cobuild leaf project log only is allowed` + ); + } else { + buildCacheTerminal.writeWarningLine( + `Failed to get log files only build cache for the project "${project.packageName}"` + ); + } } - const restoreFromCacheSuccess: boolean = await restoreCacheAsync( - cobuildLock.operationBuildCache, - cacheId - ); + cobuildLock = await this._tryGetCobuildLockAsync({ + buildCacheContext, + operationBuildCache, + cobuildConfiguration, + packageName: project.packageName, + phaseName: phase.name + }); + } + // eslint-disable-next-line require-atomic-updates -- we are mutating the build cache context intentionally + buildCacheContext.cobuildLock = cobuildLock; + + // If possible, we want to skip this operation -- either by restoring it from the + // cache, if caching is enabled, or determining that the project + // is unchanged (using the older incremental execution logic). These two approaches, + // "caching" and "skipping", are incompatible, so only one applies. + // + // Note that "caching" and "skipping" take two different approaches + // to tracking dependents: + // + // - For caching, "isCacheReadAllowed" is set if a project supports + // incremental builds, and determining whether this project or a dependent + // has changed happens inside the hashing logic. + // + + const { error: errorLogPath } = getProjectLogFilePaths({ + project, + logFilenameIdentifier: operation.logFilenameIdentifier + }); + const restoreCacheAsync = async ( + // TODO: Investigate if `operationBuildCacheForRestore` is always the same instance as `operationBuildCache` + // above, and if it is, remove this parameter + operationBuildCacheForRestore: OperationBuildCache | undefined, + specifiedCacheId?: string + ): Promise => { + buildCacheContext.isCacheReadAttempted = true; + const restoreFromCacheSuccess: boolean | undefined = + await operationBuildCacheForRestore?.tryRestoreFromCacheAsync( + buildCacheTerminal, + specifiedCacheId + ); if (restoreFromCacheSuccess) { - return status; + buildCacheContext.cacheRestored = true; + await runnerContext.runWithTerminalAsync( + async (taskTerminal, terminalProvider) => { + // Restore the original state of the operation without cache + await operationMetadataManager?.tryRestoreAsync({ + terminalProvider, + terminal: buildCacheTerminal, + errorLogPath, + cobuildContextId: cobuildConfiguration?.cobuildContextId, + cobuildRunnerId: cobuildConfiguration?.cobuildRunnerId + }); + }, + { createLogFile: false } + ); } - } else if (!buildCacheContext.isCacheReadAttempted && buildCacheContext.isCacheReadAllowed) { + return !!restoreFromCacheSuccess; + }; + if (cobuildLock) { + // handling rebuilds. "rush rebuild" or "rush retest" command will save operations to + // the build cache once completed, but does not retrieve them (since the "incremental" + // flag is disabled). However, we still need a cobuild to be able to retrieve a finished + // build from another cobuild in this case. + const cobuildCompletedState: ICobuildCompletedState | undefined = + await cobuildLock.getCompletedStateAsync(); + if (cobuildCompletedState) { + const { status, cacheId } = cobuildCompletedState; + + if (record.operation.settings?.allowCobuildWithoutCache) { + // This should only be enabled if the experiment for cobuild orchestration is enabled. + return status; + } + + const restoreFromCacheSuccess: boolean = await restoreCacheAsync( + cobuildLock.operationBuildCache, + cacheId + ); + + if (restoreFromCacheSuccess) { + return status; + } + } else if (!buildCacheContext.isCacheReadAttempted && buildCacheContext.isCacheReadAllowed) { + const restoreFromCacheSuccess: boolean = await restoreCacheAsync(operationBuildCache); + + if (restoreFromCacheSuccess) { + return OperationStatus.FromCache; + } + } + } else if (buildCacheContext.isCacheReadAllowed) { const restoreFromCacheSuccess: boolean = await restoreCacheAsync(operationBuildCache); if (restoreFromCacheSuccess) { return OperationStatus.FromCache; } } - } else if (buildCacheContext.isCacheReadAllowed) { - const restoreFromCacheSuccess: boolean = await restoreCacheAsync(operationBuildCache); - if (restoreFromCacheSuccess) { - return OperationStatus.FromCache; - } - } - - if (buildCacheContext.isCacheWriteAllowed && cobuildLock) { - const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); - if (acquireSuccess) { - const { periodicCallback } = buildCacheContext; - periodicCallback.addCallback(async () => { - await cobuildLock?.renewLockAsync(); - }); - periodicCallback.start(); - } else { - setTimeout(() => { - record.status = OperationStatus.Ready; - }, 500); - return OperationStatus.Executing; + if (buildCacheContext.isCacheWriteAllowed && cobuildLock) { + const acquireSuccess: boolean = await cobuildLock.tryAcquireLockAsync(); + if (acquireSuccess) { + const { periodicCallback } = buildCacheContext; + periodicCallback.addCallback(async () => { + await cobuildLock?.renewLockAsync(); + }); + periodicCallback.start(); + } else { + setTimeout(() => { + record.status = OperationStatus.Ready; + }, 500); + return OperationStatus.Executing; + } } - } - }; - - return await runBeforeExecute(); - } - ); - - hooks.afterExecuteOperation.tapPromise( - PLUGIN_NAME, - async (runnerContext: IOperationRunnerContext): Promise => { - const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; - const { status, stopwatch, _operationMetadataManager: operationMetadataManager, operation } = record; - - const { associatedProject: project, runner, enabled } = operation; + }; - if (!enabled || !runner?.cacheable) { - return; + return await runBeforeExecute(); } + ); - const buildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByOperation(operation); + graph.hooks.afterExecuteOperationAsync.tapPromise( + PLUGIN_NAME, + async (runnerContext: IOperationRunnerContext): Promise => { + const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; + const { + status, + stopwatch, + _operationMetadataManager: operationMetadataManager, + operation + } = record; - if (!buildCacheContext) { - return; - } + const { associatedProject: project, runner } = operation; - // No need to run for the following operation status - if (!record.isTerminal || record.status === OperationStatus.NoOp) { - return; - } + if (!record.enabled || !runner?.cacheable) { + return; + } - const { cobuildLock, operationBuildCache, isCacheWriteAllowed, buildCacheTerminal, cacheRestored } = - buildCacheContext; - - try { - if (!cacheRestored) { - // Save the metadata to disk - const { logFilenameIdentifier } = operationMetadataManager; - const { duration: durationInSeconds } = stopwatch; - const { - text: logPath, - error: errorLogPath, - jsonl: logChunksPath - } = getProjectLogFilePaths({ - project, - logFilenameIdentifier - }); - await operationMetadataManager.saveAsync({ - durationInSeconds, - cobuildContextId: cobuildLock?.cobuildConfiguration.cobuildContextId, - cobuildRunnerId: cobuildLock?.cobuildConfiguration.cobuildRunnerId, - logPath, - errorLogPath, - logChunksPath - }); + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByOperation(operation); + + if (!buildCacheContext) { + return; } - if (!buildCacheTerminal) { - // This should not happen - throw new InternalError(`Build Cache Terminal is not created`); + // No need to run for the following operation status + if (!record.isTerminal || record.status === OperationStatus.NoOp) { + return; } - let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; - let setCacheEntryPromise: (() => Promise | undefined) | undefined; - if (cobuildLock && isCacheWriteAllowed) { - const { cacheId, contextId } = cobuildLock.cobuildContext; + const { cobuildLock, operationBuildCache, isCacheWriteAllowed, buildCacheTerminal, cacheRestored } = + buildCacheContext; + + try { + if (!cacheRestored) { + // Save the metadata to disk + const { logFilenameIdentifier } = operationMetadataManager; + const { duration: durationInSeconds } = stopwatch; + const { + text: logPath, + error: errorLogPath, + jsonl: logChunksPath + } = getProjectLogFilePaths({ + project, + logFilenameIdentifier + }); + await operationMetadataManager.saveAsync({ + durationInSeconds, + cobuildContextId: cobuildLock?.cobuildConfiguration.cobuildContextId, + cobuildRunnerId: cobuildLock?.cobuildConfiguration.cobuildRunnerId, + logPath, + errorLogPath, + logChunksPath + }); + } - let finalCacheId: string = cacheId; - if (status === OperationStatus.Failure) { - finalCacheId = `${cacheId}-${contextId}-failed`; - } else if (status === OperationStatus.SuccessWithWarning && !record.runner.warningsAreAllowed) { - finalCacheId = `${cacheId}-${contextId}-warnings`; + if (!buildCacheTerminal) { + // This should not happen + throw new InternalError(`Build Cache Terminal is not created`); } - switch (status) { - case OperationStatus.SuccessWithWarning: - case OperationStatus.Success: - case OperationStatus.Failure: { - const currentStatus: ICobuildCompletedState['status'] = status; - setCompletedStatePromiseFunction = () => { - return cobuildLock?.setCompletedStateAsync({ - status: currentStatus, - cacheId: finalCacheId - }); - }; - setCacheEntryPromise = () => - cobuildLock.operationBuildCache.trySetCacheEntryAsync(buildCacheTerminal, finalCacheId); + + let setCompletedStatePromiseFunction: (() => Promise | undefined) | undefined; + let setCacheEntryPromise: (() => Promise | undefined) | undefined; + if (cobuildLock && isCacheWriteAllowed) { + const { cacheId, contextId } = cobuildLock.cobuildContext; + + let finalCacheId: string = cacheId; + if (status === OperationStatus.Failure) { + finalCacheId = `${cacheId}-${contextId}-failed`; + } else if (status === OperationStatus.SuccessWithWarning && !record.runner.warningsAreAllowed) { + finalCacheId = `${cacheId}-${contextId}-warnings`; + } + switch (status) { + case OperationStatus.SuccessWithWarning: + case OperationStatus.Success: + case OperationStatus.Failure: { + const currentStatus: ICobuildCompletedState['status'] = status; + setCompletedStatePromiseFunction = () => { + return cobuildLock?.setCompletedStateAsync({ + status: currentStatus, + cacheId: finalCacheId + }); + }; + setCacheEntryPromise = () => + cobuildLock.operationBuildCache.trySetCacheEntryAsync(buildCacheTerminal, finalCacheId); + } } } - } - const taskIsSuccessful: boolean = - status === OperationStatus.Success || - (status === OperationStatus.SuccessWithWarning && - record.runner.warningsAreAllowed && - allowWarningsInSuccessfulBuild); + const taskIsSuccessful: boolean = + status === OperationStatus.Success || + (status === OperationStatus.SuccessWithWarning && + record.runner.warningsAreAllowed && + allowWarningsInSuccessfulBuild); - // If the command is successful, we can calculate project hash, and no dependencies were skipped, - // write a new cache entry. - if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && operationBuildCache) { - setCacheEntryPromise = () => operationBuildCache.trySetCacheEntryAsync(buildCacheTerminal); - } - if (!cacheRestored) { - const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise?.(); - await setCompletedStatePromiseFunction?.(); + // If the command is successful, we can calculate project hash, and no dependencies were skipped, + // write a new cache entry. + if (!setCacheEntryPromise && taskIsSuccessful && isCacheWriteAllowed && operationBuildCache) { + setCacheEntryPromise = () => operationBuildCache.trySetCacheEntryAsync(buildCacheTerminal); + } + if (!cacheRestored) { + const cacheWriteSuccess: boolean | undefined = await setCacheEntryPromise?.(); + await setCompletedStatePromiseFunction?.(); - if (cacheWriteSuccess === false && status === OperationStatus.Success) { - record.status = OperationStatus.SuccessWithWarning; + if (cacheWriteSuccess === false && status === OperationStatus.Success) { + record.status = OperationStatus.SuccessWithWarning; + } } + } finally { + buildCacheContext.buildCacheTerminalWritable?.close(); + buildCacheContext.periodicCallback.stop(); } - } finally { - buildCacheContext.buildCacheTerminalWritable?.close(); - buildCacheContext.periodicCallback.stop(); } - } - ); + ); - hooks.afterExecuteOperation.tap( - PLUGIN_NAME, - (record: IOperationRunnerContext & IOperationExecutionResult): void => { - const { operation } = record; - const buildCacheContext: IOperationBuildCacheContext | undefined = - this._buildCacheContextByOperation.get(operation); - // Status changes to direct dependents - let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; - - switch (record.status) { - case OperationStatus.Skipped: { - // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. - blockCacheWrite = true; - break; + graph.hooks.afterExecuteOperationAsync.tap( + PLUGIN_NAME, + (record: IOperationRunnerContext & IOperationExecutionResult): void => { + const { operation } = record; + const buildCacheContext: IOperationBuildCacheContext | undefined = + this._buildCacheContextByOperation.get(operation); + // Status changes to direct dependents + let blockCacheWrite: boolean = !buildCacheContext?.isCacheWriteAllowed; + + switch (record.status) { + case OperationStatus.Skipped: { + // Skipping means cannot guarantee integrity, so prevent cache writes in dependents. + blockCacheWrite = true; + break; + } } - } - // Apply status changes to direct dependents - if (blockCacheWrite) { - for (const consumer of operation.consumers) { - const consumerBuildCacheContext: IOperationBuildCacheContext | undefined = - this._getBuildCacheContextByOperation(consumer); - if (consumerBuildCacheContext) { - consumerBuildCacheContext.isCacheWriteAllowed = false; + // Apply status changes to direct dependents + if (blockCacheWrite) { + for (const consumer of operation.consumers) { + const consumerBuildCacheContext: IOperationBuildCacheContext | undefined = + this._getBuildCacheContextByOperation(consumer); + if (consumerBuildCacheContext) { + consumerBuildCacheContext.isCacheWriteAllowed = false; + } } } } - } - ); + ); - hooks.afterExecuteOperations.tapPromise(PLUGIN_NAME, async () => { - this._buildCacheContextByOperation.clear(); + graph.hooks.afterExecuteIterationAsync.tap(PLUGIN_NAME, (status: OperationStatus) => { + this._buildCacheContextByOperation.clear(); + return status; + }); }); } diff --git a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts index e800aaaf871..770480762ce 100644 --- a/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ConsoleTimelinePlugin.ts @@ -5,15 +5,12 @@ import type { ITerminal } from '@rushstack/terminal'; import { Colorize, PrintUtilities } from '@rushstack/terminal'; import type { IPhase } from '../../api/CommandLineConfiguration'; -import type { - ICreateOperationsContext, - IPhasedCommandPlugin, - PhasedCommandHooks -} from '../../pluginFramework/PhasedCommandHooks'; -import type { IExecutionResult } from './IOperationExecutionResult'; +import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; +import type { IExecutionResult, IOperationExecutionResult } from './IOperationExecutionResult'; import { OperationStatus } from './OperationStatus'; import type { CobuildConfiguration } from '../../api/CobuildConfiguration'; import type { OperationExecutionRecord } from './OperationExecutionRecord'; +import type { Operation } from './Operation'; const PLUGIN_NAME: 'ConsoleTimelinePlugin' = 'ConsoleTimelinePlugin'; @@ -54,16 +51,22 @@ export class ConsoleTimelinePlugin implements IPhasedCommandPlugin { } public apply(hooks: PhasedCommandHooks): void { - hooks.afterExecuteOperations.tap( - PLUGIN_NAME, - (result: IExecutionResult, context: ICreateOperationsContext): void => { - _printTimeline({ - terminal: this._terminal, - result, - cobuildConfiguration: context.cobuildConfiguration - }); - } - ); + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph, context) => { + graph.hooks.afterExecuteIterationAsync.tap( + PLUGIN_NAME, + ( + status: OperationStatus, + operationResults: ReadonlyMap + ): OperationStatus => { + _printTimeline({ + terminal: this._terminal, + result: { status, operationResults }, + cobuildConfiguration: context.cobuildConfiguration + }); + return status; + } + ); + }); } } diff --git a/libraries/rush-lib/src/logic/operations/DebugHashesPlugin.ts b/libraries/rush-lib/src/logic/operations/DebugHashesPlugin.ts index 8af179838d6..343f9b906ad 100644 --- a/libraries/rush-lib/src/logic/operations/DebugHashesPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/DebugHashesPlugin.ts @@ -5,7 +5,7 @@ import { Colorize, type ITerminal } from '@rushstack/terminal'; import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import type { Operation } from './Operation'; -import type { IOperationExecutionResult } from './IOperationExecutionResult'; +import type { IConfigurableOperation, IOperationStateHashComponents } from './IOperationExecutionResult'; const PLUGIN_NAME: 'DebugHashesPlugin' = 'DebugHashesPlugin'; @@ -17,22 +17,28 @@ export class DebugHashesPlugin implements IPhasedCommandPlugin { } public apply(hooks: PhasedCommandHooks): void { - hooks.beforeExecuteOperations.tap( - PLUGIN_NAME, - (operations: Map) => { - const terminal: ITerminal = this._terminal; - terminal.writeLine(Colorize.blue(`===== Begin Hash Computation =====`)); - for (const [operation, record] of operations) { - terminal.writeLine(Colorize.cyan(`--- ${operation.name} ---`)); - record.getStateHashComponents().forEach((component) => { - terminal.writeLine(component); - }); - terminal.writeLine(Colorize.green(`Result: ${record.getStateHash()}`)); - // Add a blank line between operations to visually separate them - terminal.writeLine(); + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.configureIteration.tap( + PLUGIN_NAME, + (operations: ReadonlyMap) => { + const terminal: ITerminal = this._terminal; + terminal.writeLine(Colorize.blue(`===== Begin Hash Computation =====`)); + for (const [operation, record] of operations) { + terminal.writeLine(Colorize.cyan(`--- ${operation.name} ---`)); + const { dependencies, local, config }: IOperationStateHashComponents = + record.getStateHashComponents(); + for (const dep of dependencies) { + terminal.writeLine(dep); + } + terminal.writeLine(`local=${local}`); + terminal.writeLine(`config=${config}`); + terminal.writeLine(Colorize.green(`Result: ${record.getStateHash()}`)); + // Add a blank line between operations to visually separate them + terminal.writeLine(); + } + terminal.writeLine(Colorize.blue(`===== End Hash Computation =====`)); } - terminal.writeLine(Colorize.blue(`===== End Hash Computation =====`)); - } - ); + ); + }); } } diff --git a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts index e7658c210b8..8df76ec1b69 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationExecutionResult.ts @@ -4,19 +4,76 @@ import type { StdioSummarizer, IProblemCollector } from '@rushstack/terminal'; import type { OperationStatus } from './OperationStatus'; +import type { IOperationLastState } from './IOperationRunner'; import type { Operation } from './Operation'; import type { IStopwatchResult } from '../../utilities/Stopwatch'; import type { ILogFilePaths } from './ProjectLogWritable'; /** - * The `IOperationExecutionResult` interface represents the results of executing an {@link Operation}. + * Structured components of the state hash for an operation. + * @alpha + */ +export interface IOperationStateHashComponents { + /** + * The state hashes of operation dependencies, sorted by name. + * Each entry is of the form `{dependencyName}={hash}`. + */ + readonly dependencies: readonly string[]; + /** + * The hash of the operation's own local inputs (e.g. tracked files, environment variables). + */ + readonly local: string; + /** + * The hash of the operation's configuration (e.g. CLI parameters). + */ + readonly config: string; +} + +/** * @alpha */ -export interface IOperationExecutionResult { +export interface IBaseOperationExecutionResult { /** * The operation itself */ readonly operation: Operation; + + /** + * The relative path to the folder that contains operation metadata. This folder will be automatically included in cache entries. + */ + readonly metadataFolderPath: string; + + /** + * Gets the hash of the state of all registered inputs to this operation. + * Calling this method will throw if Git is not available. + */ + getStateHash(): string; + + /** + * Gets the structured components of the state hash. This is useful for debugging and + * incremental change detection. + * Calling this method will throw if Git is not available. + */ + getStateHashComponents(): IOperationStateHashComponents; +} + +/** + * The `IConfigurableOperation` interface represents an {@link Operation} whose + * execution can be configured before running. + * @alpha + */ +export interface IConfigurableOperation extends IBaseOperationExecutionResult { + /** + * True if the operation should execute in this iteration, false otherwise. + */ + enabled: boolean; +} + +/** + * The `IOperationExecutionResult` interface represents the results of executing an {@link Operation}. + * @alpha + */ +export interface IOperationExecutionResult extends IBaseOperationExecutionResult, IOperationLastState { /** * The current execution status of an operation. Operations start in the 'ready' state, * but can be 'blocked' if an upstream operation failed. It is 'executing' when @@ -33,6 +90,10 @@ export interface IOperationExecutionResult { * If this operation is only present in the graph to maintain dependency relationships, this flag will be set to true. */ readonly silent: boolean; + /** + * True if the operation should execute in this iteration, false otherwise. + */ + readonly enabled: boolean; /** * Object tracking execution timing. */ @@ -49,26 +110,10 @@ export interface IOperationExecutionResult { * The value indicates the duration of the same operation without cache hit. */ readonly nonCachedDurationMs: number | undefined; - /** - * The relative path to the folder that contains operation metadata. This folder will be automatically included in cache entries. - */ - readonly metadataFolderPath: string | undefined; /** * The paths to the log files, if applicable. */ readonly logFilePaths: ILogFilePaths | undefined; - - /** - * Gets the hash of the state of all registered inputs to this operation. - * Calling this method will throw if Git is not available. - */ - getStateHash(): string; - - /** - * Gets the components of the state hash. This is useful for debugging purposes. - * Calling this method will throw if Git is not available. - */ - getStateHashComponents(): ReadonlyArray; } /** diff --git a/libraries/rush-lib/src/logic/operations/IOperationGraph.ts b/libraries/rush-lib/src/logic/operations/IOperationGraph.ts new file mode 100644 index 00000000000..294a6205675 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/IOperationGraph.ts @@ -0,0 +1,166 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { TerminalWritable } from '@rushstack/terminal'; + +import type { Operation } from './Operation'; +import type { IOperationExecutionResult } from './IOperationExecutionResult'; +import type { Parallelism } from './ParseParallelism'; +import type { OperationStatus } from './OperationStatus'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import type { OperationGraphHooks } from '../../pluginFramework/OperationGraphHooks'; + +/** + * Options for a single iteration of operation execution. + * @alpha + */ +export interface IOperationGraphIterationOptions { + inputsSnapshot?: IInputsSnapshot; + + /** + * The time when the iteration was scheduled, if available, as returned by `performance.now()`. + */ + startTime?: number; +} + +/** + * Public API for the operation graph. + * @alpha + */ +export interface IOperationGraph { + /** + * Hooks into the execution process for operations + */ + readonly hooks: OperationGraphHooks; + + /** + * The set of operations in the graph. + */ + readonly operations: ReadonlySet; + + /** + * A map from each `Operation` in the graph to its current result record. + * The map is updated in real time as operations execute during an iteration. + * Only statuses representing a completed execution (e.g. `Success`, `Failure`, + * `SuccessWithWarning`) write to this map; statuses such as `Skipped` or `Aborted` — + * which indicate that an operation did not actually run — do not update it. + * For operations that have not yet run in the current iteration, the map retains the + * result from whichever prior iteration the operation last ran in. + * An entry with status `Ready` indicates that the operation is considered stale and + * has been queued to run again. + * Empty until at least one operation has completed execution. + */ + readonly resultByOperation: ReadonlyMap; + + /** + * The maximum allowed parallelism for this operation graph. + * Reads as a concrete integer. Accepts a `Parallelism` value and coerces it on write. + */ + get parallelism(): number; + set parallelism(value: Parallelism); + + /** + * If additional debug information should be printed during execution. + */ + debugMode: boolean; + + /** + * If true, operations will be executed in "quiet mode" where only errors are reported. + */ + quietMode: boolean; + + /** + * If true, allow operations to oversubscribe the CPU. Defaults to true. + */ + allowOversubscription: boolean; + + /** + * When true, the operation graph will pause before running the next iteration (manual mode). + * When false, iterations run automatically when scheduled. + */ + pauseNextIteration: boolean; + + /** + * The current overall status of the execution. + */ + readonly status: OperationStatus; + + /** + * The current set of terminal destinations. + */ + readonly terminalDestinations: ReadonlySet; + + /** + * True if there is a scheduled (but not yet executing) iteration. + * This will be false while an iteration is actively executing, or when no work is scheduled. + */ + readonly hasScheduledIteration: boolean; + + /** + * AbortController controlling the lifetime of the overall session (e.g. watch mode). + * Aborting this controller should signal all listeners (such as file system watchers) to dispose + * and prevent further iterations from being scheduled. + */ + readonly abortController: AbortController; + + /** + * Abort the current execution iteration, if any. Operations that have already started + * will run to completion; only operations that have not yet begun will be aborted. + */ + abortCurrentIterationAsync(): Promise; + + /** + * Cleans up any resources used by the operation runners, if applicable. + * @param operations - The operations whose runners should be closed, or undefined to close all runners. + */ + closeRunnersAsync(operations?: Iterable): Promise; + + /** + * Executes a single iteration of the operations. + * @param options - Options for this execution iteration. + * @returns A promise that resolves to true if the iteration has work to be done, or false if the iteration was empty and therefore not scheduled. + */ + scheduleIterationAsync(options: IOperationGraphIterationOptions): Promise; + + /** + * Executes all operations in the currently scheduled iteration, if any. + * @returns A promise which is resolved when all operations have been processed to a final state. + */ + executeScheduledIterationAsync(): Promise; + + /** + * Invalidates the specified operations, causing them to be re-executed. + * @param operations - The operations to invalidate, or undefined to invalidate all operations. + * @param reason - Optional reason for invalidation. + */ + invalidateOperations(operations?: Iterable, reason?: string): void; + + /** + * Sets the enabled state for a collection of operations. + * + * @param operations - The operations whose enabled state should be updated. + * @param targetState - The target enabled state to apply. + * @param mode - 'unsafe' to directly mutate only the provided operations, 'safe' to also enable + * transitive dependencies of enabled operations and disable transitive dependents of disabled operations. + * @returns true if any operation's enabled state changed, false otherwise. + */ + setEnabledStates( + operations: Iterable, + targetState: Operation['enabled'], + mode: 'safe' | 'unsafe' + ): boolean; + + /** + * Adds a terminal destination for output. Only new output will be sent to the destination. + * @param destination - The destination to add. + */ + addTerminalDestination(destination: TerminalWritable): void; + + /** + * Removes a terminal destination for output. Optionally closes the stream. + * New output will no longer be sent to the destination. + * @param destination - The destination to remove. + * @param close - Whether to close the stream. Defaults to `true`. + */ + removeTerminalDestination(destination: TerminalWritable, close?: boolean): boolean; +} diff --git a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts index d6c907bf65a..e8dc3f2d10d 100644 --- a/libraries/rush-lib/src/logic/operations/IOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IOperationRunner.ts @@ -9,6 +9,18 @@ import type { OperationMetadataManager } from './OperationMetadataManager'; import type { IStopwatchResult } from '../../utilities/Stopwatch'; import type { IEnvironment } from '../../utilities/Utilities'; +/** + * A snapshot of a previous operation execution, passed to runners to inform incremental behavior. + * + * @beta + */ +export interface IOperationLastState { + /** + * The status from the previous execution of this operation. + */ + readonly status: OperationStatus; +} + /** * Information passed to the executing `IOperationRunner` * @@ -57,6 +69,13 @@ export interface IOperationRunnerContext { */ error?: Error; + /** + * Returns a callback that invalidates this operation so that it will be re-executed in the next iteration. + * The returned callback captures only the minimal state needed, avoiding retention of the full context. + * Callers should store the result rather than calling this method repeatedly. + */ + getInvalidateCallback(): (reason: string) => void; + /** * Invokes the specified callback with a terminal that is associated with this operation. * @@ -73,7 +92,7 @@ export interface IOperationRunnerContext { /** * The `Operation` class is a node in the dependency graph of work that needs to be scheduled by the - * `OperationExecutionManager`. Each `Operation` has a `runner` member of type `IOperationRunner`, whose + * `OperationGraph`. Each `Operation` has a `runner` member of type `IOperationRunner`, whose * implementation manages the actual process for running a single operation. * * @beta @@ -111,13 +130,28 @@ export interface IOperationRunner { */ readonly isNoOp?: boolean; + /** + * If true, this runner currently owns some kind of active resource (e.g. a service or a watch process). + * This can be used to determine if the operation is "in progress" even if it is not currently executing. + * If the runner supports this property, it should update it as appropriate during execution. + * The property is optional to avoid breaking existing implementations of IOperationRunner. + */ + readonly isActive?: boolean; + /** * Method to be executed for the operation. + * @param context - The context object containing information about the execution environment. + * @param lastState - The last execution result of this operation, if any. */ - executeAsync(context: IOperationRunnerContext): Promise; + executeAsync(context: IOperationRunnerContext, lastState?: IOperationLastState): Promise; /** * Return a hash of the configuration that affects the operation. */ getConfigHash(): string; + + /** + * If this runner performs any background work to optimize future runs, this method will clean it up. + */ + closeAsync?(): Promise; } diff --git a/libraries/rush-lib/src/logic/operations/IPCOperationRunner.ts b/libraries/rush-lib/src/logic/operations/IPCOperationRunner.ts index 6fbc924d87e..1d4637e6cd3 100644 --- a/libraries/rush-lib/src/logic/operations/IPCOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/IPCOperationRunner.ts @@ -9,8 +9,7 @@ import type { IRequestRunEventMessage, ISyncEventMessage, IRunCommandMessage, - IExitCommandMessage, - OperationRequestRunCallback + IExitCommandMessage } from '@rushstack/operation-graph'; import { TerminalProviderSeverity, type ITerminal, type ITerminalProvider } from '@rushstack/terminal'; @@ -18,7 +17,7 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { Utilities } from '../../utilities/Utilities'; -import type { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; +import type { IOperationRunner, IOperationRunnerContext, IOperationLastState } from './IOperationRunner'; import { OperationError } from './OperationError'; import { OperationStatus } from './OperationStatus'; @@ -26,10 +25,10 @@ export interface IIPCOperationRunnerOptions { phase: IPhase; project: RushConfigurationProject; name: string; - commandToRun: string; + initialCommand: string; + incrementalCommand: string | undefined; commandForHash: string; persist: boolean; - requestRun: OperationRequestRunCallback; ignoredParameterValues: ReadonlyArray; } @@ -56,10 +55,10 @@ export class IPCOperationRunner implements IOperationRunner { public readonly warningsAreAllowed: boolean; private readonly _rushProject: RushConfigurationProject; - private readonly _commandToRun: string; + private readonly _initialCommand: string; + private readonly _incrementalCommand: string | undefined; private readonly _commandForHash: string; private readonly _persist: boolean; - private readonly _requestRun: OperationRequestRunCallback; private readonly _ignoredParameterValues: ReadonlyArray; private _ipcProcess: ChildProcess | undefined; @@ -72,15 +71,25 @@ export class IPCOperationRunner implements IOperationRunner { options.phase.allowWarningsOnSuccess || false; this._rushProject = options.project; - this._commandToRun = options.commandToRun; + this._initialCommand = options.initialCommand; + this._incrementalCommand = options.incrementalCommand; this._commandForHash = options.commandForHash; this._persist = options.persist; - this._requestRun = options.requestRun; this._ignoredParameterValues = options.ignoredParameterValues; } - public async executeAsync(context: IOperationRunnerContext): Promise { + public get isActive(): boolean { + return !!(this._ipcProcess && !this._ipcProcess.killed && typeof this._ipcProcess.exitCode !== 'number'); + } + + public async executeAsync( + context: IOperationRunnerContext, + lastState?: IOperationLastState + ): Promise { + const commandToRun: string = + lastState && this._incrementalCommand ? this._incrementalCommand : this._initialCommand; + const invalidate: (reason: string) => void = context.getInvalidateCallback(); return await context.runWithTerminalAsync( async (terminal: ITerminal, terminalProvider: ITerminalProvider): Promise => { let isConnected: boolean = false; @@ -93,13 +102,13 @@ export class IPCOperationRunner implements IOperationRunner { } // Run the operation - terminal.writeLine('Invoking: ' + this._commandToRun); + terminal.writeLine('Invoking: ' + commandToRun); const { rushConfiguration, projectFolder } = this._rushProject; const { environment: initialEnvironment } = context; - this._ipcProcess = Utilities.executeLifecycleCommandAsync(this._commandToRun, { + this._ipcProcess = Utilities.executeLifecycleCommandAsync(commandToRun, { rushConfiguration, workingDirectory: projectFolder, initCwd: rushConfiguration.commonTempFolder, @@ -120,7 +129,10 @@ export class IPCOperationRunner implements IOperationRunner { this._ipcProcess.on('message', (message: unknown) => { if (isRequestRunEventMessage(message)) { - this._requestRun(message.requestor, message.detail); + const reason: string = message.detail + ? `${message.requestor}: ${message.detail}` + : message.requestor; + invalidate(reason); } else if (isSyncEventMessage(message)) { resolveReadyPromise(); } @@ -194,7 +206,7 @@ export class IPCOperationRunner implements IOperationRunner { }); if (isConnected && !this._persist) { - await this.shutdownAsync(); + await this.closeAsync(); } // @rushstack/operation-graph does not currently have a concept of "Success with Warning" @@ -213,7 +225,7 @@ export class IPCOperationRunner implements IOperationRunner { return this._commandForHash; } - public async shutdownAsync(): Promise { + public async closeAsync(): Promise { const { _ipcProcess: subProcess } = this; if (!subProcess) { return; diff --git a/libraries/rush-lib/src/logic/operations/IPCOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/IPCOperationRunnerPlugin.ts index f87dcb1685a..7d5cccd7851 100644 --- a/libraries/rush-lib/src/logic/operations/IPCOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/IPCOperationRunnerPlugin.ts @@ -6,10 +6,8 @@ import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; -import type { IOperationExecutionResult } from './IOperationExecutionResult'; import { IPCOperationRunner } from './IPCOperationRunner'; import type { Operation } from './Operation'; -import { OperationStatus } from './OperationStatus'; import { PLUGIN_NAME as ShellOperationPluginName, formatCommand, @@ -25,26 +23,17 @@ const PLUGIN_NAME: 'IPCOperationRunnerPlugin' = 'IPCOperationRunnerPlugin'; */ export class IPCOperationRunnerPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { - // Workaround until the operation graph persists for the lifetime of the watch process - const runnerCache: Map = new Map(); - - const operationStatesByRunner: WeakMap = new WeakMap(); - - let currentContext: ICreateOperationsContext | undefined; - - hooks.createOperations.tapPromise( + hooks.createOperationsAsync.tap( { name: PLUGIN_NAME, before: ShellOperationPluginName }, - async (operations: Set, context: ICreateOperationsContext) => { - const { isWatch, isInitial } = context; - if (!isWatch) { + (operations: Set, context: ICreateOperationsContext) => { + const { isWatch, isIncrementalBuildAllowed } = context; + if (!isWatch || !isIncrementalBuildAllowed) { return operations; } - currentContext = context; - const getCustomParameterValues: (operation: Operation) => ICustomParameterValuesForOperation = getCustomParameterValuesByOperation(); @@ -62,79 +51,45 @@ export class IPCOperationRunnerPlugin implements IPhasedCommandPlugin { const { name: phaseName } = phase; - const rawScript: string | undefined = - (!isInitial ? scripts[`${phaseName}:incremental:ipc`] : undefined) ?? scripts[`${phaseName}:ipc`]; + const incrementalScript: string | undefined = scripts[`${phaseName}:incremental:ipc`]; + let initialScript: string | undefined = scripts[`${phaseName}:ipc`]; - if (!rawScript) { + // Both must be absent to skip. A project may define only one of `_phase:ipc` or + // `_phase:incremental:ipc` — defining either opts into the IPC runner. + if (!initialScript && !incrementalScript) { continue; } + initialScript ??= scripts[phaseName]; + // This is the command that will be used to identify the cache entry for this operation, to allow // for this operation (or downstream operations) to be restored from the build cache. const commandForHash: string | undefined = phase.shellCommand ?? scripts?.[phaseName]; const { parameterValues: customParameterValues, ignoredParameterValues } = getCustomParameterValues(operation); - const commandToRun: string = formatCommand(rawScript, customParameterValues); + const initialCommand: string = formatCommand(initialScript, customParameterValues); + const incrementalCommand: string | undefined = incrementalScript + ? formatCommand(incrementalScript, customParameterValues) + : undefined; const operationName: string = getDisplayName(phase, project); - let maybeIpcOperationRunner: IPCOperationRunner | undefined = runnerCache.get(operationName); - if (!maybeIpcOperationRunner) { - const ipcOperationRunner: IPCOperationRunner = (maybeIpcOperationRunner = new IPCOperationRunner({ - phase, - project, - name: operationName, - commandToRun, - commandForHash, - persist: true, - ignoredParameterValues, - requestRun: (requestor: string, detail?: string) => { - const operationState: IOperationExecutionResult | undefined = - operationStatesByRunner.get(ipcOperationRunner); - if (!operationState) { - return; - } - - const status: OperationStatus = operationState.status; - if ( - status === OperationStatus.Waiting || - status === OperationStatus.Ready || - status === OperationStatus.Queued - ) { - // Already pending. No-op. - return; - } - - currentContext?.invalidateOperation?.( - operation, - detail ? `${requestor}: ${detail}` : requestor - ); - } - })); - runnerCache.set(operationName, ipcOperationRunner); - } - - operation.runner = maybeIpcOperationRunner; + const ipcOperationRunner: IPCOperationRunner = new IPCOperationRunner({ + phase, + project, + name: operationName, + initialCommand, + incrementalCommand, + commandForHash, + persist: true, + ignoredParameterValues + }); + + operation.runner = ipcOperationRunner; } return operations; } ); - - hooks.beforeExecuteOperations.tap( - PLUGIN_NAME, - (records: Map, context: ICreateOperationsContext) => { - currentContext = context; - for (const [{ runner }, result] of records) { - if (runner instanceof IPCOperationRunner) { - operationStatesByRunner.set(runner, result); - } - } - } - ); - - hooks.shutdownAsync.tapPromise(PLUGIN_NAME, async () => { - await Promise.all(Array.from(runnerCache.values(), (runner) => runner.shutdownAsync())); - }); } } diff --git a/libraries/rush-lib/src/logic/operations/IgnoredParametersPlugin.ts b/libraries/rush-lib/src/logic/operations/IgnoredParametersPlugin.ts index 0de6aa422f6..5ac89f7f733 100644 --- a/libraries/rush-lib/src/logic/operations/IgnoredParametersPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/IgnoredParametersPlugin.ts @@ -20,20 +20,22 @@ export const RUSHSTACK_CLI_IGNORED_PARAMETER_NAMES_ENV_VAR: 'RUSHSTACK_CLI_IGNOR */ export class IgnoredParametersPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { - hooks.createEnvironmentForOperation.tap( - PLUGIN_NAME, - (env: IEnvironment, record: IOperationExecutionResult) => { - const { settings } = record.operation; + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.createEnvironmentForOperation.tap( + PLUGIN_NAME, + (env: IEnvironment, record: IOperationExecutionResult) => { + const { settings } = record.operation; - // If there are parameter names to ignore, set the environment variable - if (settings?.parameterNamesToIgnore && settings.parameterNamesToIgnore.length > 0) { - env[RUSHSTACK_CLI_IGNORED_PARAMETER_NAMES_ENV_VAR] = JSON.stringify( - settings.parameterNamesToIgnore - ); - } + // If there are parameter names to ignore, set the environment variable + if (settings?.parameterNamesToIgnore && settings.parameterNamesToIgnore.length > 0) { + env[RUSHSTACK_CLI_IGNORED_PARAMETER_NAMES_ENV_VAR] = JSON.stringify( + settings.parameterNamesToIgnore + ); + } - return env; - } - ); + return env; + } + ); + }); } } diff --git a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts index 86f899453f4..846362409c5 100644 --- a/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/LegacySkipPlugin.ts @@ -8,11 +8,8 @@ import { PrintUtilities, Colorize, type ITerminal } from '@rushstack/terminal'; import type { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; -import type { - IExecuteOperationsContext, - IPhasedCommandPlugin, - PhasedCommandHooks -} from '../../pluginFramework/PhasedCommandHooks'; +import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; +import type { IOperationGraphIterationOptions } from './IOperationGraph'; import type { IOperationRunnerContext } from './IOperationRunner'; import type { IOperationExecutionResult } from './IOperationExecutionResult'; @@ -66,198 +63,200 @@ export class LegacySkipPlugin implements IPhasedCommandPlugin { const { terminal, changedProjectsOnly, isIncrementalBuildAllowed, allowWarningsInSuccessfulBuild } = this._options; - hooks.beforeExecuteOperations.tap( - PLUGIN_NAME, - ( - operations: ReadonlyMap, - context: IExecuteOperationsContext - ): void => { - let logGitWarning: boolean = false; - const { inputsSnapshot } = context; - - for (const record of operations.values()) { - const { operation } = record; - const { associatedProject, associatedPhase, runner, logFilenameIdentifier } = operation; - if (!runner) { - continue; - } - - if (!runner.cacheable) { - stateMap.set(operation, { - allowSkip: true, - packageDeps: undefined, - packageDepsPath: '' - }); - continue; - } - - const packageDepsFilename: string = `package-deps_${logFilenameIdentifier}.json`; + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.beforeExecuteIterationAsync.tap( + PLUGIN_NAME, + ( + operations: ReadonlyMap, + iterationOptions: IOperationGraphIterationOptions + ): void => { + let logGitWarning: boolean = false; + const { inputsSnapshot } = iterationOptions; + + for (const record of operations.values()) { + const { operation } = record; + const { associatedProject, associatedPhase, runner, logFilenameIdentifier } = operation; + if (!runner) { + continue; + } - const packageDepsPath: string = path.join( - associatedProject.projectRushTempFolder, - packageDepsFilename - ); + if (!runner.cacheable) { + stateMap.set(operation, { + allowSkip: true, + packageDeps: undefined, + packageDepsPath: '' + }); + continue; + } - let packageDeps: IProjectDeps | undefined; + const packageDepsFilename: string = `package-deps_${logFilenameIdentifier}.json`; - try { - const fileHashes: ReadonlyMap | undefined = - inputsSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name); + const packageDepsPath: string = path.join( + associatedProject.projectRushTempFolder, + packageDepsFilename + ); - if (!fileHashes) { - logGitWarning = true; - continue; + let packageDeps: IProjectDeps | undefined; + + try { + const fileHashes: ReadonlyMap | undefined = + inputsSnapshot?.getTrackedFileHashesForOperation(associatedProject, associatedPhase.name); + + if (!fileHashes) { + logGitWarning = true; + continue; + } + + const files: Record = {}; + for (const [filePath, fileHash] of fileHashes) { + files[filePath] = fileHash; + } + + packageDeps = { + files, + arguments: runner.getConfigHash() + }; + } catch (error) { + // To test this code path: + // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" + terminal.writeLine( + `Unable to calculate incremental state for ${record.operation.name}: ` + + (error as Error).toString() + ); + terminal.writeLine( + Colorize.cyan('Rush will proceed without incremental execution and change detection.') + ); } - const files: Record = {}; - for (const [filePath, fileHash] of fileHashes) { - files[filePath] = fileHash; - } + stateMap.set(operation, { + packageDepsPath, + packageDeps, + allowSkip: isIncrementalBuildAllowed + }); + } - packageDeps = { - files, - arguments: runner.getConfigHash() - }; - } catch (error) { + if (logGitWarning) { // To test this code path: - // Delete a project's ".rush/temp/shrinkwrap-deps.json" then run "rush build --verbose" - terminal.writeLine( - `Unable to calculate incremental state for ${record.operation.name}: ` + - (error as Error).toString() - ); + // Remove the `.git` folder then run "rush build --verbose" terminal.writeLine( - Colorize.cyan('Rush will proceed without incremental execution and change detection.') + Colorize.cyan( + PrintUtilities.wrapWords( + 'This workspace does not appear to be tracked by Git. ' + + 'Rush will proceed without incremental execution, caching, and change detection.' + ) + ) ); } - - stateMap.set(operation, { - packageDepsPath, - packageDeps, - allowSkip: isIncrementalBuildAllowed - }); } + ); - if (logGitWarning) { - // To test this code path: - // Remove the `.git` folder then run "rush build --verbose" - terminal.writeLine( - Colorize.cyan( - PrintUtilities.wrapWords( - 'This workspace does not appear to be tracked by Git. ' + - 'Rush will proceed without incremental execution, caching, and change detection.' - ) - ) - ); - } - } - ); - - hooks.beforeExecuteOperation.tapPromise( - PLUGIN_NAME, - async ( - record: IOperationRunnerContext & IOperationExecutionResult - ): Promise => { - const { operation } = record; - const skipRecord: ILegacySkipRecord | undefined = stateMap.get(operation); - if (!skipRecord) { - // This operation doesn't support skip detection. - return; - } + graph.hooks.beforeExecuteOperationAsync.tapPromise( + PLUGIN_NAME, + async ( + record: IOperationRunnerContext & IOperationExecutionResult + ): Promise => { + const { operation } = record; + const skipRecord: ILegacySkipRecord | undefined = stateMap.get(operation); + if (!skipRecord) { + // This operation doesn't support skip detection. + return; + } - if (!operation.runner!.cacheable) { - // This operation doesn't support skip detection. - return; - } + if (!operation.runner!.cacheable) { + // This operation doesn't support skip detection. + return; + } - const { associatedProject } = operation; + const { associatedProject } = operation; - const { packageDepsPath, packageDeps, allowSkip } = skipRecord; + const { packageDepsPath, packageDeps, allowSkip } = skipRecord; - let lastProjectDeps: IProjectDeps | undefined = undefined; + let lastProjectDeps: IProjectDeps | undefined = undefined; - try { - const lastDepsContents: string = await FileSystem.readFileAsync(packageDepsPath); - lastProjectDeps = JSON.parse(lastDepsContents); - } catch (e) { - if (!FileSystem.isNotExistError(e)) { - // Warn and ignore - treat failing to load the file as the operation being not built. - // TODO: Update this to be the terminal specific to the operation. - terminal.writeWarningLine( - `Warning: error parsing ${packageDepsPath}: ${e}. Ignoring and treating this operation as not run.` - ); + try { + const lastDepsContents: string = await FileSystem.readFileAsync(packageDepsPath); + lastProjectDeps = JSON.parse(lastDepsContents); + } catch (e) { + if (!FileSystem.isNotExistError(e)) { + // Warn and ignore - treat failing to load the file as the operation being not built. + // TODO: Update this to be the terminal specific to the operation. + terminal.writeWarningLine( + `Warning: error parsing ${packageDepsPath}: ${e}. Ignoring and treating this operation as not run.` + ); + } } - } - if (allowSkip) { - const isPackageUnchanged: boolean = !!( - lastProjectDeps && - packageDeps && - packageDeps.arguments === lastProjectDeps.arguments && - _areShallowEqual(packageDeps.files, lastProjectDeps.files) - ); + if (allowSkip) { + const isPackageUnchanged: boolean = !!( + lastProjectDeps && + packageDeps && + packageDeps.arguments === lastProjectDeps.arguments && + _areShallowEqual(packageDeps.files, lastProjectDeps.files) + ); - if (isPackageUnchanged) { - return OperationStatus.Skipped; + if (isPackageUnchanged) { + return OperationStatus.Skipped; + } } - } - // TODO: Remove legacyDepsPath with the next major release of Rush - const legacyDepsPath: string = path.join(associatedProject.projectFolder, 'package-deps.json'); + // TODO: Remove legacyDepsPath with the next major release of Rush + const legacyDepsPath: string = path.join(associatedProject.projectFolder, 'package-deps.json'); - await Promise.all([ - // Delete the legacy package-deps.json - FileSystem.deleteFileAsync(legacyDepsPath), + await Promise.all([ + // Delete the legacy package-deps.json + FileSystem.deleteFileAsync(legacyDepsPath), - // If the deps file exists, remove it before starting execution. - FileSystem.deleteFileAsync(packageDepsPath) - ]); - } - ); + // If the deps file exists, remove it before starting execution. + FileSystem.deleteFileAsync(packageDepsPath) + ]); + } + ); - hooks.afterExecuteOperation.tapPromise( - PLUGIN_NAME, - async (record: IOperationRunnerContext & IOperationExecutionResult): Promise => { - const { status, operation } = record; + graph.hooks.afterExecuteOperationAsync.tapPromise( + PLUGIN_NAME, + async (record: IOperationRunnerContext & IOperationExecutionResult): Promise => { + const { status, operation } = record; - const skipRecord: ILegacySkipRecord | undefined = stateMap.get(operation); - if (!skipRecord) { - return; - } + const skipRecord: ILegacySkipRecord | undefined = stateMap.get(operation); + if (!skipRecord) { + return; + } - const blockSkip: boolean = - !skipRecord.allowSkip || - (!changedProjectsOnly && - (status === OperationStatus.Success || status === OperationStatus.SuccessWithWarning)); - if (blockSkip) { - for (const consumer of operation.consumers) { - const consumerSkipRecord: ILegacySkipRecord | undefined = stateMap.get(consumer); - if (consumerSkipRecord) { - consumerSkipRecord.allowSkip = false; + const blockSkip: boolean = + !skipRecord.allowSkip || + (!changedProjectsOnly && + (status === OperationStatus.Success || status === OperationStatus.SuccessWithWarning)); + if (blockSkip) { + for (const consumer of operation.consumers) { + const consumerSkipRecord: ILegacySkipRecord | undefined = stateMap.get(consumer); + if (consumerSkipRecord) { + consumerSkipRecord.allowSkip = false; + } } } - } - if (!record.operation.runner!.cacheable) { - // This operation doesn't support skip detection. - return; - } + if (!record.operation.runner!.cacheable) { + // This operation doesn't support skip detection. + return; + } - const { packageDeps, packageDepsPath } = skipRecord; - - if ( - status === OperationStatus.NoOp || - (packageDeps && - (status === OperationStatus.Success || - (status === OperationStatus.SuccessWithWarning && - record.operation.runner!.warningsAreAllowed && - allowWarningsInSuccessfulBuild))) - ) { - // Write deps on success. - await JsonFile.saveAsync(packageDeps, packageDepsPath, { - ensureFolderExists: true - }); + const { packageDeps, packageDepsPath } = skipRecord; + + if ( + status === OperationStatus.NoOp || + (packageDeps && + (status === OperationStatus.Success || + (status === OperationStatus.SuccessWithWarning && + record.operation.runner!.warningsAreAllowed && + allowWarningsInSuccessfulBuild))) + ) { + // Write deps on success. + await JsonFile.saveAsync(packageDeps, packageDepsPath, { + ensureFolderExists: true + }); + } } - } - ); + ); + }); } } diff --git a/libraries/rush-lib/src/logic/operations/NodeDiagnosticDirPlugin.ts b/libraries/rush-lib/src/logic/operations/NodeDiagnosticDirPlugin.ts index 28df11044c4..70e1d88c69f 100644 --- a/libraries/rush-lib/src/logic/operations/NodeDiagnosticDirPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/NodeDiagnosticDirPlugin.ts @@ -39,25 +39,27 @@ export class NodeDiagnosticDirPlugin implements IPhasedCommandPlugin { return diagnosticDir; }; - hooks.createEnvironmentForOperation.tap( - PLUGIN_NAME, - (env: IEnvironment, record: IOperationExecutionResult) => { - const diagnosticDir: string | undefined = getDiagnosticDir(record.operation); - if (!diagnosticDir) { - return env; - } + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.createEnvironmentForOperation.tap( + PLUGIN_NAME, + (env: IEnvironment, record: IOperationExecutionResult) => { + const diagnosticDir: string | undefined = getDiagnosticDir(record.operation); + if (!diagnosticDir) { + return env; + } - // Not all versions of NodeJS create the directory, so ensure it exists: - FileSystem.ensureFolder(diagnosticDir); + // Not all versions of NodeJS create the directory, so ensure it exists: + FileSystem.ensureFolder(diagnosticDir); - const { NODE_OPTIONS } = env; + const { NODE_OPTIONS } = env; - const diagnosticDirEnv: string = `--diagnostic-dir="${diagnosticDir}"`; + const diagnosticDirEnv: string = `--diagnostic-dir="${diagnosticDir}"`; - env.NODE_OPTIONS = NODE_OPTIONS ? `${NODE_OPTIONS} ${diagnosticDirEnv}` : diagnosticDirEnv; + env.NODE_OPTIONS = NODE_OPTIONS ? `${NODE_OPTIONS} ${diagnosticDirEnv}` : diagnosticDirEnv; - return env; - } - ); + return env; + } + ); + }); } } diff --git a/libraries/rush-lib/src/logic/operations/Operation.ts b/libraries/rush-lib/src/logic/operations/Operation.ts index be3ec8ac5fc..62360a61532 100644 --- a/libraries/rush-lib/src/logic/operations/Operation.ts +++ b/libraries/rush-lib/src/logic/operations/Operation.ts @@ -5,6 +5,19 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProjec import type { IPhase } from '../../api/CommandLineConfiguration'; import type { IOperationRunner } from './IOperationRunner'; import type { IOperationSettings } from '../../api/RushProjectConfiguration'; +import { type Parallelism, parseParallelismPercent } from './ParseParallelism'; + +/** + * State for the `enabled` property of an `Operation`. + * + * - `true`: The operation should be executed if it or any dependencies changed. + * - `false`: The operation should be skipped. + * - `"ignore-dependency-changes"`: The operation should be executed if there are local changes in the project, + * otherwise it should be skipped. This is useful for operations like "test" where you may want to skip + * testing projects that haven't changed. + * @alpha + */ +export type OperationEnabledState = boolean | 'ignore-dependency-changes'; /** * Options for constructing a new Operation. @@ -21,6 +34,19 @@ export interface IOperationOptions { */ project: RushConfigurationProject; + /** + * If set to false, this operation will be skipped during evaluation (return OperationStatus.Skipped). + * This is useful for plugins to alter the scope of the operation graph across executions, + * e.g. to enable or disable unit test execution, or to include or exclude dependencies. + * + * The special value "ignore-dependency-changes" can be used to indicate that this operation should only + * be executed if there are local changes in the project. This is useful for operations like + * "test" where you may want to skip testing projects that haven't changed. + * + * The default value is `true`, meaning the operation will be executed if it or any dependencies change. + */ + enabled?: OperationEnabledState; + /** * When the scheduler is ready to process this `Operation`, the `runner` implements the actual work of * running the operation. @@ -40,7 +66,7 @@ export interface IOperationOptions { /** * The `Operation` class is a node in the dependency graph of work that needs to be scheduled by the - * `OperationExecutionManager`. Each `Operation` has a `runner` member of type `IOperationRunner`, whose + * `OperationGraph`. Each `Operation` has a `runner` member of type `IOperationRunner`, whose * implementation manages the actual process of running a single operation. * * The graph of `Operation` instances will be cloned into a separate execution graph after processing. @@ -82,17 +108,23 @@ export class Operation { public runner: IOperationRunner | undefined = undefined; /** - * The weight for this operation. This scalar is the contribution of this operation to the - * `criticalPathLength` calculation above. Modify to indicate the following: - * - `weight` === 1: indicates that this operation has an average duration - * - `weight` > 1: indicates that this operation takes longer than average and so the scheduler - * should try to favor starting it over other, shorter operations. An example might be an operation that - * bundles an entire application and runs whole-program optimization. - * - `weight` < 1: indicates that this operation takes less time than average and so the scheduler - * should favor other, longer operations over it. An example might be an operation to unpack a cached - * output, or an operation using NullOperationRunner, which might use a value of 0. + * The concurrency weight for this operation. When coerced to an integer via `coerceParallelism`, + * this value represents how many concurrency slots the operation consumes while running. + * + * May be specified as: + * - A raw `number`: used directly as the slot count (e.g. `2` consumes two slots). + * - An `IParallelismScalar` (e.g. `{ scalar: 0.5 }`): coerced relative to the graph's + * configured `maxParallelism` at execution time, so the weight scales with the available + * concurrency rather than being fixed at parse time. + * + * Coerced values guide scheduling as follows: + * - `1` slot: typical operation consuming one logical thread. + * - `> 1` slots: operation that spawns multiple threads or requires significant RAM; reserving + * extra slots prevents overloading the machine (e.g. a whole-program bundler or a test suite + * that runs its own internal parallelism). + * - `0` slots: effectively free (e.g. a no-op or cache-restore step). */ - public weight: number = 1; + public weight: Parallelism; /** * Get the operation settings for this operation, defaults to the values defined in @@ -104,17 +136,27 @@ export class Operation { * If set to false, this operation will be skipped during evaluation (return OperationStatus.Skipped). * This is useful for plugins to alter the scope of the operation graph across executions, * e.g. to enable or disable unit test execution, or to include or exclude dependencies. + * + * The special value "ignore-dependency-changes" can be used to indicate that this operation should only + * be executed if there are local changes in the project. This is useful for operations like + * "test" where you may want to skip testing projects that haven't changed. + * + * The default value is `true`, meaning the operation will be executed if it or any dependencies change. */ - public enabled: boolean; + public enabled: OperationEnabledState; public constructor(options: IOperationOptions) { - const { phase, project, runner, settings, logFilenameIdentifier } = options; + const { phase, project, runner, settings, logFilenameIdentifier, enabled = true } = options; this.associatedPhase = phase; this.associatedProject = project; this.runner = runner; this.settings = settings; this.logFilenameIdentifier = logFilenameIdentifier; - this.enabled = true; + this.enabled = enabled; + this.weight = _getFinalWeight( + settings?.weight ?? 1, + runner?.name ?? `${project.packageName} (${phase.name}` + ); } /** @@ -157,3 +199,16 @@ export class Operation { (dependency.consumers as Set).delete(this); } } + +function _getFinalWeight(rawWeight: string | number, context: string): Parallelism { + if (typeof rawWeight === 'number') { + // Explicit numeric weight allows any value. + return rawWeight; + } else { + try { + return { scalar: parseParallelismPercent(rawWeight) }; + } catch (err) { + throw new Error(`Invalid weight for operation "${context}": ${err.message}`); + } + } +} diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts deleted file mode 100644 index 976a214f026..00000000000 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionManager.ts +++ /dev/null @@ -1,477 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { - type TerminalWritable, - StdioWritable, - TextRewriterTransform, - Colorize, - ConsoleTerminalProvider, - TerminalChunkKind -} from '@rushstack/terminal'; -import { StreamCollator, type CollatedTerminal, type CollatedWriter } from '@rushstack/stream-collator'; -import { NewlineKind, Async, InternalError, AlreadyReportedError } from '@rushstack/node-core-library'; - -import { AsyncOperationQueue, type IOperationSortFunction } from './AsyncOperationQueue'; -import type { Operation } from './Operation'; -import { OperationStatus } from './OperationStatus'; -import { type IOperationExecutionRecordContext, OperationExecutionRecord } from './OperationExecutionRecord'; -import type { IExecutionResult } from './IOperationExecutionResult'; -import type { IEnvironment } from '../../utilities/Utilities'; -import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; -import type { IStopwatchResult } from '../../utilities/Stopwatch'; - -export interface IOperationExecutionManagerOptions { - quietMode: boolean; - debugMode: boolean; - parallelism: number; - allowOversubscription: boolean; - inputsSnapshot?: IInputsSnapshot; - destination?: TerminalWritable; - - beforeExecuteOperationAsync?: (operation: OperationExecutionRecord) => Promise; - afterExecuteOperationAsync?: (operation: OperationExecutionRecord) => Promise; - createEnvironmentForOperation?: (operation: OperationExecutionRecord) => IEnvironment; - onOperationStatusChangedAsync?: (record: OperationExecutionRecord) => void; - beforeExecuteOperationsAsync?: (records: Map) => Promise; -} - -/** - * Format "======" lines for a shell window with classic 80 columns - */ -const ASCII_HEADER_WIDTH: number = 79; - -const prioritySort: IOperationSortFunction = ( - a: OperationExecutionRecord, - b: OperationExecutionRecord -): number => { - return a.criticalPathLength! - b.criticalPathLength!; -}; - -/** - * Sorts operations lexicographically by their name. - * @param a - The first operation to compare - * @param b - The second operation to compare - * @returns A comparison result: -1 if a < b, 0 if a === b, 1 if a > b - */ -function sortOperationsByName(a: Operation, b: Operation): number { - const aName: string = a.name; - const bName: string = b.name; - return aName === bName ? 0 : aName < bName ? -1 : 1; -} - -/** - * A class which manages the execution of a set of tasks with interdependencies. - * Initially, and at the end of each task execution, all unblocked tasks - * are added to a ready queue which is then executed. This is done continually until all - * tasks are complete, or prematurely fails if any of the tasks fail. - */ -export class OperationExecutionManager { - private readonly _executionRecords: Map; - private readonly _quietMode: boolean; - private readonly _parallelism: number; - private readonly _allowOversubscription: boolean; - private readonly _totalOperations: number; - - private readonly _outputWritable: TerminalWritable; - private readonly _colorsNewlinesTransform: TextRewriterTransform; - private readonly _streamCollator: StreamCollator; - - private readonly _terminal: CollatedTerminal; - - private readonly _beforeExecuteOperation?: ( - operation: OperationExecutionRecord - ) => Promise; - private readonly _afterExecuteOperation?: (operation: OperationExecutionRecord) => Promise; - private readonly _onOperationStatusChanged?: (record: OperationExecutionRecord) => void; - private readonly _beforeExecuteOperations?: ( - records: Map - ) => Promise; - private readonly _createEnvironmentForOperation?: (operation: OperationExecutionRecord) => IEnvironment; - - // Variables for current status - private _hasAnyFailures: boolean; - private _hasAnyNonAllowedWarnings: boolean; - private _hasAnyAborted: boolean; - private _completedOperations: number; - private _executionQueue: AsyncOperationQueue; - - public constructor(operations: Set, options: IOperationExecutionManagerOptions) { - const { - quietMode, - debugMode, - parallelism, - allowOversubscription, - inputsSnapshot, - beforeExecuteOperationAsync: beforeExecuteOperation, - afterExecuteOperationAsync: afterExecuteOperation, - onOperationStatusChangedAsync: onOperationStatusChanged, - beforeExecuteOperationsAsync: beforeExecuteOperations, - createEnvironmentForOperation - } = options; - this._completedOperations = 0; - this._quietMode = quietMode; - this._hasAnyFailures = false; - this._hasAnyNonAllowedWarnings = false; - this._hasAnyAborted = false; - this._parallelism = parallelism; - this._allowOversubscription = allowOversubscription; - - this._beforeExecuteOperation = beforeExecuteOperation; - this._afterExecuteOperation = afterExecuteOperation; - this._beforeExecuteOperations = beforeExecuteOperations; - this._createEnvironmentForOperation = createEnvironmentForOperation; - this._onOperationStatusChanged = (record: OperationExecutionRecord) => { - if (record.status === OperationStatus.Ready) { - this._executionQueue.assignOperations(); - } - onOperationStatusChanged?.(record); - }; - - // TERMINAL PIPELINE: - // - // streamCollator --> colorsNewlinesTransform --> StdioWritable - // - this._outputWritable = options.destination || StdioWritable.instance; - this._colorsNewlinesTransform = new TextRewriterTransform({ - destination: this._outputWritable, - normalizeNewlines: NewlineKind.OsDefault, - removeColors: !ConsoleTerminalProvider.supportsColor - }); - this._streamCollator = new StreamCollator({ - destination: this._colorsNewlinesTransform, - onWriterActive: this._streamCollator_onWriterActive - }); - this._terminal = this._streamCollator.terminal; - - // Convert the developer graph to the mutable execution graph - const executionRecordContext: IOperationExecutionRecordContext = { - streamCollator: this._streamCollator, - onOperationStatusChanged: this._onOperationStatusChanged, - createEnvironment: this._createEnvironmentForOperation, - inputsSnapshot, - debugMode, - quietMode - }; - - // Sort the operations by name to ensure consistency and readability. - const sortedOperations: Operation[] = Array.from(operations).sort(sortOperationsByName); - - let totalOperations: number = 0; - const executionRecords: Map = (this._executionRecords = new Map()); - for (const operation of sortedOperations) { - const executionRecord: OperationExecutionRecord = new OperationExecutionRecord( - operation, - executionRecordContext - ); - - executionRecords.set(operation, executionRecord); - if (!executionRecord.silent) { - // Only count non-silent operations - totalOperations++; - } - } - this._totalOperations = totalOperations; - - for (const [operation, record] of executionRecords) { - for (const dependency of operation.dependencies) { - const dependencyRecord: OperationExecutionRecord | undefined = executionRecords.get(dependency); - if (!dependencyRecord) { - throw new Error( - `Operation "${record.name}" declares a dependency on operation "${dependency.name}" that is not in the set of operations to execute.` - ); - } - record.dependencies.add(dependencyRecord); - dependencyRecord.consumers.add(record); - } - } - - // Ensure we compute the compute the state hashes for all operations before the runtime graph potentially mutates. - if (inputsSnapshot) { - for (const record of executionRecords.values()) { - record.getStateHash(); - } - } - - const executionQueue: AsyncOperationQueue = new AsyncOperationQueue( - this._executionRecords.values(), - prioritySort - ); - this._executionQueue = executionQueue; - } - - private _streamCollator_onWriterActive = (writer: CollatedWriter | undefined): void => { - if (writer) { - this._completedOperations++; - - // Format a header like this - // - // ==[ @rushstack/the-long-thing ]=================[ 1 of 1000 ]== - - // leftPart: "==[ @rushstack/the-long-thing " - const leftPart: string = Colorize.gray('==[') + ' ' + Colorize.cyan(writer.taskName) + ' '; - const leftPartLength: number = 4 + writer.taskName.length + 1; - - // rightPart: " 1 of 1000 ]==" - const completedOfTotal: string = `${this._completedOperations} of ${this._totalOperations}`; - const rightPart: string = ' ' + Colorize.white(completedOfTotal) + ' ' + Colorize.gray(']=='); - const rightPartLength: number = 1 + completedOfTotal.length + 4; - - // middlePart: "]=================[" - const twoBracketsLength: number = 2; - const middlePartLengthMinusTwoBrackets: number = Math.max( - ASCII_HEADER_WIDTH - (leftPartLength + rightPartLength + twoBracketsLength), - 0 - ); - - const middlePart: string = Colorize.gray(']' + '='.repeat(middlePartLengthMinusTwoBrackets) + '['); - - this._terminal.writeStdoutLine('\n' + leftPart + middlePart + rightPart); - - if (!this._quietMode) { - this._terminal.writeStdoutLine(''); - } - } - }; - - /** - * Executes all operations which have been registered, returning a promise which is resolved when all the - * operations are completed successfully, or rejects when any operation fails. - */ - public async executeAsync(abortController: AbortController): Promise { - this._completedOperations = 0; - const totalOperations: number = this._totalOperations; - const abortSignal: AbortSignal = abortController.signal; - - if (!this._quietMode) { - const plural: string = totalOperations === 1 ? '' : 's'; - this._terminal.writeStdoutLine(`Selected ${totalOperations} operation${plural}:`); - const nonSilentOperations: string[] = []; - for (const record of this._executionRecords.values()) { - if (!record.silent) { - nonSilentOperations.push(record.name); - } - } - nonSilentOperations.sort(); - for (const name of nonSilentOperations) { - this._terminal.writeStdoutLine(` ${name}`); - } - this._terminal.writeStdoutLine(''); - } - - // For display purposes, cap the reported number of simultaneous processes by the number of operations. - // This avoids confusing messages like "Executing a maximum of 10 simultaneous processes..." when - // there are only 4 operations. - const maxSimultaneousProcesses: number = Math.min(totalOperations, this._parallelism); - this._terminal.writeStdoutLine( - `Executing a maximum of ${maxSimultaneousProcesses} simultaneous processes...` - ); - - await this._beforeExecuteOperations?.(this._executionRecords); - - // This function is a callback because it may write to the collatedWriter before - // operation.executeAsync returns (and cleans up the writer) - const onOperationCompleteAsync: (record: OperationExecutionRecord) => Promise = async ( - record: OperationExecutionRecord - ) => { - // If the operation is not terminal, we should _only_ notify the queue to assign operations. - if (!record.isTerminal) { - this._executionQueue.assignOperations(); - } else { - try { - await this._afterExecuteOperation?.(record); - } catch (e) { - this._reportOperationErrorIfAny(record); - record.error = e; - record.status = OperationStatus.Failure; - } - this._onOperationComplete(record); - } - }; - - const onOperationStartAsync: ( - record: OperationExecutionRecord - ) => Promise = async (record: OperationExecutionRecord) => { - return await this._beforeExecuteOperation?.(record); - }; - - await Async.forEachAsync( - this._executionQueue, - async (record: OperationExecutionRecord) => { - if (abortSignal.aborted) { - record.status = OperationStatus.Aborted; - // Bypass the normal completion handler, directly mark the operation as aborted and unblock the queue. - // We do this to ensure that we aren't messing with the stopwatch or terminal. - this._hasAnyAborted = true; - this._executionQueue.complete(record); - } else { - await record.executeAsync({ - onStart: onOperationStartAsync, - onResult: onOperationCompleteAsync - }); - } - }, - { - allowOversubscription: this._allowOversubscription, - // In weighted mode, concurrency represents the total "unit budget", not the max number of tasks. - // Do not cap by totalOperations, since that would incorrectly shrink the unit budget and - // reduce parallelism for operations with weight > 1. - concurrency: this._parallelism, - weighted: true - } - ); - - const status: OperationStatus = this._hasAnyFailures - ? OperationStatus.Failure - : this._hasAnyAborted - ? OperationStatus.Aborted - : this._hasAnyNonAllowedWarnings - ? OperationStatus.SuccessWithWarning - : OperationStatus.Success; - - return { - operationResults: this._executionRecords, - status - }; - } - - private _reportOperationErrorIfAny(record: OperationExecutionRecord): void { - // Failed operations get reported, even if silent. - // Generally speaking, silent operations shouldn't be able to fail, so this is a safety measure. - let message: string | undefined = undefined; - if (record.error) { - if (!(record.error instanceof AlreadyReportedError)) { - message = record.error.message; - } - } - - if (message) { - // This creates the writer, so don't do this until needed - record.collatedWriter.terminal.writeStderrLine(message); - // Ensure that the summary isn't blank if we have an error message - // If the summary already contains max lines of stderr, this will get dropped, so we hope those lines - // are more useful than the final exit code. - record.stdioSummarizer.writeChunk({ - text: `${message}\n`, - kind: TerminalChunkKind.Stdout - }); - } - } - - /** - * Handles the result of the operation and propagates any relevant effects. - */ - private _onOperationComplete(record: OperationExecutionRecord): void { - const { runner, name, status, silent, _operationMetadataManager: operationMetadataManager } = record; - const stopwatch: IStopwatchResult = - operationMetadataManager?.tryRestoreStopwatch(record.stopwatch) || record.stopwatch; - - switch (status) { - /** - * This operation failed. Mark it as such and all reachable dependents as blocked. - */ - case OperationStatus.Failure: { - // Failed operations get reported, even if silent. - // Generally speaking, silent operations shouldn't be able to fail, so this is a safety measure. - this._reportOperationErrorIfAny(record); - - // This creates the writer, so don't do this globally - const { terminal } = record.collatedWriter; - terminal.writeStderrLine(Colorize.red(`"${name}" failed to build.`)); - const blockedQueue: Set = new Set(record.consumers); - - for (const blockedRecord of blockedQueue) { - if (blockedRecord.status === OperationStatus.Waiting) { - // Now that we have the concept of architectural no-ops, we could implement this by replacing - // {blockedRecord.runner} with a no-op that sets status to Blocked and logs the blocking - // operations. However, the existing behavior is a bit simpler, so keeping that for now. - if (!blockedRecord.silent) { - terminal.writeStdoutLine(`"${blockedRecord.name}" is blocked by "${name}".`); - } - blockedRecord.status = OperationStatus.Blocked; - - this._executionQueue.complete(blockedRecord); - if (!blockedRecord.silent) { - // Only increment the count if the operation is not silent to avoid confusing the user. - // The displayed total is the count of non-silent operations. - this._completedOperations++; - } - - for (const dependent of blockedRecord.consumers) { - blockedQueue.add(dependent); - } - } else if (blockedRecord.status !== OperationStatus.Blocked) { - // It shouldn't be possible for operations to be in any state other than Waiting or Blocked - throw new InternalError( - `Blocked operation ${blockedRecord.name} is in an unexpected state: ${blockedRecord.status}` - ); - } - } - this._hasAnyFailures = true; - break; - } - - /** - * This operation was restored from the build cache. - */ - case OperationStatus.FromCache: { - if (!silent) { - record.collatedWriter.terminal.writeStdoutLine( - Colorize.green(`"${name}" was restored from the build cache.`) - ); - } - break; - } - - /** - * This operation was skipped via legacy change detection. - */ - case OperationStatus.Skipped: { - if (!silent) { - record.collatedWriter.terminal.writeStdoutLine(Colorize.green(`"${name}" was skipped.`)); - } - break; - } - - /** - * This operation intentionally didn't do anything. - */ - case OperationStatus.NoOp: { - if (!silent) { - record.collatedWriter.terminal.writeStdoutLine(Colorize.gray(`"${name}" did not define any work.`)); - } - break; - } - - case OperationStatus.Success: { - if (!silent) { - record.collatedWriter.terminal.writeStdoutLine( - Colorize.green(`"${name}" completed successfully in ${stopwatch.toString()}.`) - ); - } - break; - } - - case OperationStatus.SuccessWithWarning: { - if (!silent) { - record.collatedWriter.terminal.writeStderrLine( - Colorize.yellow(`"${name}" completed with warnings in ${stopwatch.toString()}.`) - ); - } - this._hasAnyNonAllowedWarnings = this._hasAnyNonAllowedWarnings || !runner.warningsAreAllowed; - break; - } - - case OperationStatus.Aborted: { - this._hasAnyAborted ||= true; - break; - } - - default: { - throw new InternalError(`Unexpected operation status: ${status}`); - } - } - - this._executionQueue.complete(record); - } -} diff --git a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts index 57a6d1a17ec..2de7655d1d3 100644 --- a/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts +++ b/libraries/rush-lib/src/logic/operations/OperationExecutionRecord.ts @@ -18,6 +18,7 @@ import { import { InternalError, NewlineKind, FileError } from '@rushstack/node-core-library'; import { CollatedTerminal, type CollatedWriter, type StreamCollator } from '@rushstack/stream-collator'; +import { coerceParallelism } from './ParseParallelism'; import { OperationStatus, TERMINAL_STATUSES } from './OperationStatus'; import type { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; import type { Operation } from './Operation'; @@ -26,8 +27,9 @@ import { OperationMetadataManager } from './OperationMetadataManager'; import type { IPhase } from '../../api/CommandLineConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { CollatedTerminalProvider } from '../../utilities/CollatedTerminalProvider'; -import type { IOperationExecutionResult } from './IOperationExecutionResult'; +import type { IOperationExecutionResult, IOperationStateHashComponents } from './IOperationExecutionResult'; import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import { OperationError } from './OperationError'; import { RushConstants } from '../RushConstants'; import type { IEnvironment } from '../../utilities/Utilities'; import { @@ -41,14 +43,25 @@ import { */ export interface IOperationExecutionRecordContext { streamCollator: StreamCollator; - onOperationStatusChanged?: (record: OperationExecutionRecord) => void; + onOperationStateChanged?: (record: OperationExecutionRecord) => void; createEnvironment?: (record: OperationExecutionRecord) => IEnvironment; + invalidate?: (operations: Iterable, reason: string) => void; inputsSnapshot: IInputsSnapshot | undefined; + maxParallelism: number; debugMode: boolean; quietMode: boolean; } +/** + * Context object for the executeAsync() method. + * @internal + */ +export interface IOperationExecutionContext { + onStartAsync: (record: OperationExecutionRecord) => Promise; + onResultAsync: (record: OperationExecutionRecord) => Promise; +} + /** * Internal class representing everything about executing an operation * @@ -66,6 +79,11 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera */ public error: Error | undefined = undefined; + /** + * If true, this operation should be executed. If false, it should be skipped. + */ + public enabled: boolean; + /** * This number represents how far away this Operation is from the furthest "root" operation (i.e. * an operation with no consumers). This helps us to calculate the critical path (i.e. the @@ -129,6 +147,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera }); public readonly runner: IOperationRunner; + public readonly weight: number; public readonly associatedPhase: IPhase; public readonly associatedProject: RushConfigurationProject; public readonly _operationMetadataManager: OperationMetadataManager; @@ -140,10 +159,10 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera private _collatedWriter: CollatedWriter | undefined = undefined; private _status: OperationStatus; private _stateHash: string | undefined; - private _stateHashComponents: ReadonlyArray | undefined; + private _stateHashComponents: IOperationStateHashComponents | undefined; public constructor(operation: Operation, context: IOperationExecutionRecordContext) { - const { runner, associatedPhase, associatedProject } = operation; + const { runner, associatedPhase, associatedProject, enabled } = operation; if (!runner) { throw new InternalError( @@ -152,7 +171,9 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera } this.operation = operation; + this.enabled = !!enabled; this.runner = runner; + this.weight = runner.isNoOp ? 0 : coerceParallelism(operation.weight, context.maxParallelism); this.associatedPhase = associatedPhase; this.associatedProject = associatedProject; this.logFilePaths = undefined; @@ -171,10 +192,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera return this.runner.name; } - public get weight(): number { - return this.operation.weight; - } - public get debugMode(): boolean { return this._context.debugMode; } @@ -205,8 +222,17 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera return this._context.createEnvironment?.(this); } - public get metadataFolderPath(): string | undefined { - return this._operationMetadataManager?.metadataFolderPath; + public getInvalidateCallback(): (reason: string) => void { + const invalidateFn: ((operations: Iterable, reason: string) => void) | undefined = + this._context.invalidate; + const operations: [Operation] = [this.operation]; + return (reason: string) => { + invalidateFn?.(operations, reason); + }; + } + + public get metadataFolderPath(): string { + return this._operationMetadataManager.metadataFolderPath; } public get isTerminal(): boolean { @@ -227,21 +253,23 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera return; } this._status = newStatus; - this._context.onOperationStatusChanged?.(this); + this._context.onOperationStateChanged?.(this); } public get silent(): boolean { - return !this.operation.enabled || this.runner.silent; + return !this.enabled || this.runner.silent; } public getStateHash(): string { if (this._stateHash === undefined) { - const components: readonly string[] = this.getStateHashComponents(); + const { dependencies, local, config } = this.getStateHashComponents(); const hasher: crypto.Hash = crypto.createHash('sha1'); - components.forEach((component) => { - hasher.update(`${RushConstants.hashDelimiter}${component}`); - }); + for (const dep of dependencies) { + hasher.update(`${RushConstants.hashDelimiter}${dep}`); + } + hasher.update(`${RushConstants.hashDelimiter}local=${local}`); + hasher.update(`${RushConstants.hashDelimiter}config=${config}`); const hash: string = hasher.digest('hex'); this._stateHash = hash; @@ -249,7 +277,7 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera return this._stateHash; } - public getStateHashComponents(): ReadonlyArray { + public getStateHashComponents(): IOperationStateHashComponents { if (!this._stateHashComponents) { const { inputsSnapshot } = this._context; @@ -265,8 +293,8 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera // The final state hashes of operation dependencies are factored into the hash to ensure that any // state changes in dependencies will invalidate the cache. - const components: string[] = Array.from(this.dependencies, (record) => { - return `${RushConstants.hashDelimiter}${record.name}=${record.getStateHash()}`; + const dependencies: string[] = Array.from(this.dependencies, (record) => { + return `${record.name}=${record.getStateHash()}`; }).sort(); const { associatedProject, associatedPhase } = this; @@ -275,17 +303,13 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera // - Git hashes of tracked files in the associated project // - Git hash of the shrinkwrap file for the project // - Git hashes of any files specified in `dependsOnAdditionalFiles` (must not be associated with a project) - const localStateHash: string = inputsSnapshot.getOperationOwnStateHash( - associatedProject, - associatedPhase.name - ); - components.push(`${RushConstants.hashDelimiter}local=${localStateHash}`); + const local: string = inputsSnapshot.getOperationOwnStateHash(associatedProject, associatedPhase.name); // Examples of data in the config hash: // - CLI parameters (ShellOperationRunner) - const configHash: string = this.runner.getConfigHash(); - components.push(`${RushConstants.hashDelimiter}config=${configHash}`); - this._stateHashComponents = components; + const config: string = this.runner.getConfigHash(); + + this._stateHashComponents = { dependencies, local, config }; } return this._stateHashComponents; } @@ -309,9 +333,6 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera logFilenameIdentifier: `${this._operationMetadataManager.logFilenameIdentifier}${logFileSuffix}` }) : undefined; - if (logFilePaths !== undefined) { - this.logFilePaths = logFilePaths; - } const projectLogWritable: TerminalWritable | undefined = logFilePaths ? await initializeProjectLogFilesAsync({ @@ -319,6 +340,11 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera enableChunkedOutput: true }) : undefined; + if (logFilePaths) { + // Only assign if it won't clear an existing value; stopgap until we support multiple sets of log files per operation. + this.logFilePaths = logFilePaths; + this._context.onOperationStateChanged?.(this); + } try { //#region OPERATION LOGGING @@ -385,13 +411,10 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera } } - public async executeAsync({ - onStart, - onResult - }: { - onStart: (record: OperationExecutionRecord) => Promise; - onResult: (record: OperationExecutionRecord) => Promise; - }): Promise { + public async executeAsync( + lastState: OperationExecutionRecord | undefined, + executeContext: IOperationExecutionContext + ): Promise { if (!this.isTerminal) { this.stopwatch.reset(); } @@ -399,15 +422,15 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera this.status = OperationStatus.Executing; try { - const earlyReturnStatus: OperationStatus | undefined = await onStart(this); + const earlyReturnStatus: OperationStatus | undefined = await executeContext.onStartAsync(this); // When the operation status returns by the hook, bypass the runner execution. if (earlyReturnStatus) { this.status = earlyReturnStatus; } else { // If the operation is disabled, skip the runner and directly mark as Skipped. // However, if the operation is a NoOp, return NoOp so that cache entries can still be written. - this.status = this.operation.enabled - ? await this.runner.executeAsync(this) + this.status = this.enabled + ? await this.runner.executeAsync(this, lastState) : this.runner.isNoOp ? OperationStatus.NoOp : OperationStatus.Skipped; @@ -415,14 +438,15 @@ export class OperationExecutionRecord implements IOperationRunnerContext, IOpera // Make sure that the stopwatch is stopped before reporting the result, otherwise endTime is undefined. this.stopwatch.stop(); // Delegate global state reporting - await onResult(this); + await executeContext.onResultAsync(this); } catch (error) { this.status = OperationStatus.Failure; - this.error = error; + this.error = + error instanceof OperationError ? error : new OperationError('executing', (error as Error).message); // Make sure that the stopwatch is stopped before reporting the result, otherwise endTime is undefined. this.stopwatch.stop(); // Delegate global state reporting - await onResult(this); + await executeContext.onResultAsync(this); } finally { if (this.isTerminal) { this._collatedWriter?.close(); diff --git a/libraries/rush-lib/src/logic/operations/OperationGraph.ts b/libraries/rush-lib/src/logic/operations/OperationGraph.ts new file mode 100644 index 00000000000..55aab7c18e4 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/OperationGraph.ts @@ -0,0 +1,1251 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + type TerminalWritable, + TextRewriterTransform, + Colorize, + ConsoleTerminalProvider, + TerminalChunkKind, + SplitterTransform +} from '@rushstack/terminal'; +import { StreamCollator, CollatedTerminal, type CollatedWriter } from '@rushstack/stream-collator'; +import { NewlineKind, Async, InternalError, AlreadyReportedError } from '@rushstack/node-core-library'; + +import { AsyncOperationQueue, type IOperationSortFunction } from './AsyncOperationQueue'; +import type { Operation } from './Operation'; +import { OperationStatus, TERMINAL_STATUSES } from './OperationStatus'; +import { + type IOperationExecutionContext, + type IOperationExecutionRecordContext, + OperationExecutionRecord +} from './OperationExecutionRecord'; +import type { IExecutionResult } from './IOperationExecutionResult'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; +import type { IEnvironment } from '../../utilities/Utilities'; +import type { IStopwatchResult } from '../../utilities/Stopwatch'; +import type { IOperationGraph, IOperationGraphIterationOptions } from './IOperationGraph'; +import { OperationGraphHooks } from '../../pluginFramework/OperationGraphHooks'; +import { type Parallelism, coerceParallelism, getNumberOfCores } from './ParseParallelism'; +import { measureAsyncFn, measureFn } from '../../utilities/performance'; +import type { ITelemetryData, ITelemetryOperationResult } from '../Telemetry'; + +export interface IOperationGraphTelemetry { + initialExtraData: Record; + changedProjectsOnlyKey: string | undefined; + nameForLog: string; + log: (telemetry: ITelemetryData) => void; +} + +export interface IOperationGraphOptions { + quietMode: boolean; + debugMode: boolean; + parallelism: Parallelism; + allowOversubscription: boolean; + destinations: Iterable; + /** Optional maximum allowed parallelism. Defaults to `getNumberOfCores()`. */ + maxParallelism?: number; + + /** + * Controller used to signal abortion of the entire execution session (e.g. terminating watch mode). + * Consumers (e.g. ProjectWatcher) can subscribe to this to perform cleanup. + */ + abortController: AbortController; + + isWatch?: boolean; + pauseNextIteration?: boolean; + + telemetry?: IOperationGraphTelemetry; + getInputsSnapshotAsync?: () => Promise; +} + +/** + * Internal context state used during an execution iteration. + */ +interface IStatefulExecutionContext { + hasAnyFailures: boolean; + hasAnyNonAllowedWarnings: boolean; + hasAnyAborted: boolean; + + executionQueue: AsyncOperationQueue; + resultByOperation: Map; + + get completedOperations(): number; + set completedOperations(value: number); +} + +/** + * Context for a single execution iteration. + */ +interface IExecutionIterationContext extends IOperationExecutionRecordContext { + abortController: AbortController; + terminal: CollatedTerminal; + + records: Map; + promise: Promise | undefined; + + startTime?: number; + + completedOperations: number; + totalOperations: number; +} + +/** + * Telemetry data for a phased execution + */ +interface IPhasedExecutionTelemetry { + [key: string]: string | number | boolean; + isInitial: boolean; + isWatch: boolean; + + countAll: number; + countSuccess: number; + countSuccessWithWarnings: number; + countFailure: number; + countBlocked: number; + countFromCache: number; + countSkipped: number; + countNoOp: number; + countAborted: number; +} + +const PERF_PREFIX: 'rush:executionManager' = 'rush:executionManager'; + +/** + * Format "======" lines for a shell window with classic 80 columns + */ +const ASCII_HEADER_WIDTH: number = 79; + +const prioritySort: IOperationSortFunction = ( + a: OperationExecutionRecord, + b: OperationExecutionRecord +): number => { + return a.criticalPathLength! - b.criticalPathLength!; +}; + +/** + * Sorts operations lexicographically by their name. + * @param a - The first operation to compare + * @param b - The second operation to compare + * @returns A comparison result: -1 if a < b, 0 if a === b, 1 if a > b + */ +function sortOperationsByName(a: Operation, b: Operation): number { + const aName: string = a.name; + const bName: string = b.name; + return aName === bName ? 0 : aName < bName ? -1 : 1; +} + +/** + * A class which manages the execution of a set of tasks with interdependencies. + */ +export class OperationGraph implements IOperationGraph { + public readonly hooks: OperationGraphHooks = new OperationGraphHooks(); + public readonly operations: Set; + public readonly abortController: AbortController; + private readonly _sortedOperations: readonly Operation[]; + + public resultByOperation: Map; + + // Mutable properties extracted from options + private _parallelism: number; + private _maxParallelism: number; + private _debugMode: boolean; + private _quietMode: boolean; + private _allowOversubscription: boolean; + private _pauseNextIteration: boolean; + + // Immutable properties from options + private readonly _isWatch: boolean; + private readonly _telemetry: IOperationGraphTelemetry | undefined; + private readonly _getInputsSnapshotAsync: (() => Promise) | undefined; + + /** + * Records invalidated during the current iteration that could not be marked `Ready` immediately + * because their record object is shared between `resultByOperation` and the active iteration's + * `records` map (mutating it mid-iteration would corrupt the summarizer's view of results). + * Maps each record to the invalidation reason; applied once the iteration completes. + */ + private readonly _deferredInvalidations: Map = new Map(); + + private _currentIteration: IExecutionIterationContext | undefined = undefined; + private _scheduledIteration: IExecutionIterationContext | undefined = undefined; + + private _terminalSplitter: SplitterTransform; + private _idleTimeout: NodeJS.Timeout | undefined = undefined; + /** Tracks if a graph state change notification has been scheduled for next tick. */ + private _graphStateChangeScheduled: boolean = false; + private _status: OperationStatus = OperationStatus.Ready; + + public constructor(operations: Set, options: IOperationGraphOptions) { + const { + debugMode, + quietMode, + parallelism, + allowOversubscription, + destinations, + maxParallelism = getNumberOfCores(), + abortController, + isWatch = false, + pauseNextIteration = false, + telemetry, + getInputsSnapshotAsync + } = options; + + this.operations = operations; + + this._maxParallelism = maxParallelism; + this._parallelism = coerceParallelism(parallelism, maxParallelism, 1); + this._debugMode = debugMode; + this._quietMode = quietMode; + this._allowOversubscription = allowOversubscription; + this._pauseNextIteration = pauseNextIteration; + this._isWatch = isWatch; + this._telemetry = telemetry; + this._getInputsSnapshotAsync = getInputsSnapshotAsync; + + this._sortedOperations = Array.from(operations).sort(sortOperationsByName); + this._terminalSplitter = new SplitterTransform({ destinations }); + this.resultByOperation = new Map(); + this.abortController = abortController; + + this.abortController.signal.addEventListener( + 'abort', + () => { + if (this._idleTimeout) { + clearTimeout(this._idleTimeout); + } + void this.closeRunnersAsync(); + }, + { once: true } + ); + } + + /** + * {@inheritDoc IOperationGraph.setEnabledStates} + */ + public setEnabledStates( + operations: Iterable, + targetState: Operation['enabled'], + mode: 'safe' | 'unsafe' + ): boolean { + const changedOperations: Set = new Set(); + const requested: Set = new Set(operations); + if (requested.size === 0) { + return false; + } + + if (mode === 'unsafe') { + for (const op of requested) { + if (op.enabled !== targetState) { + op.enabled = targetState; + changedOperations.add(op); + } + } + } else { + // Safe mode logic + if (targetState === true) { + // Expand dependencies of all provided operations (closure) + for (const op of requested) { + for (const dep of op.dependencies) { + requested.add(dep); + } + } + for (const op of requested) { + if (op.enabled !== true) { + op.enabled = true; + changedOperations.add(op); + } + } + } else if (targetState === false) { + const operationsToDisable: Set = new Set(requested); + for (const op of operationsToDisable) { + for (const dep of op.dependencies) { + operationsToDisable.add(dep); + } + } + + const enabledOperations: Set = new Set(); + for (const op of this.operations) { + if (op.enabled !== false && !operationsToDisable.has(op)) { + enabledOperations.add(op); + } + } + for (const op of enabledOperations) { + for (const dep of op.dependencies) { + enabledOperations.add(dep); + } + } + for (const op of enabledOperations) { + operationsToDisable.delete(op); + } + for (const op of operationsToDisable) { + if (op.enabled !== false) { + op.enabled = false; + changedOperations.add(op); + } + } + } else if (targetState === 'ignore-dependency-changes') { + const toEnable: Set = new Set(requested); + for (const op of toEnable) { + for (const dep of op.dependencies) { + toEnable.add(dep); + } + } + for (const op of toEnable) { + const opTargetState: Operation['enabled'] = op.settings?.ignoreChangedProjectsOnlyFlag + ? true + : targetState; + if (op.enabled !== opTargetState) { + op.enabled = opTargetState; + changedOperations.add(op); + } + } + } + } + + if (changedOperations.size > 0) { + // Notify via dedicated hook (do not schedule generic graph state change) + this.hooks.onEnableStatesChanged.call(changedOperations); + } + return changedOperations.size > 0; + } + + public get parallelism(): number { + return this._parallelism; + } + public set parallelism(value: Parallelism) { + const coerced: number = coerceParallelism(value, this._maxParallelism, 1); + if (coerced !== this._parallelism) { + this._parallelism = coerced; + this._scheduleManagerStateChanged(); + } + } + + public get debugMode(): boolean { + return this._debugMode; + } + public set debugMode(value: boolean) { + if (value !== this._debugMode) { + this._debugMode = value; + this._scheduleManagerStateChanged(); + } + } + + public get quietMode(): boolean { + return this._quietMode; + } + public set quietMode(value: boolean) { + if (value !== this._quietMode) { + this._quietMode = value; + this._scheduleManagerStateChanged(); + } + } + + public get allowOversubscription(): boolean { + return this._allowOversubscription; + } + public set allowOversubscription(value: boolean) { + if (value !== this._allowOversubscription) { + this._allowOversubscription = value; + this._scheduleManagerStateChanged(); + } + } + + public get pauseNextIteration(): boolean { + return this._pauseNextIteration; + } + public set pauseNextIteration(value: boolean) { + if (value !== this._pauseNextIteration) { + this._pauseNextIteration = value; + this._scheduleManagerStateChanged(); + + this._setIdleTimeout(); + } + } + + public get hasScheduledIteration(): boolean { + return !!this._scheduledIteration; + } + + public get status(): OperationStatus { + return this._status; + } + + public get terminalDestinations(): ReadonlySet { + return this._terminalSplitter.destinations; + } + + private _setStatus(newStatus: OperationStatus): void { + if (this._status !== newStatus) { + this._status = newStatus; + this._scheduleManagerStateChanged(); + } + } + + private _setScheduledIteration(iteration: IExecutionIterationContext | undefined): void { + const hadScheduled: boolean = !!this._scheduledIteration; + this._scheduledIteration = iteration; + if (hadScheduled !== !!this._scheduledIteration) { + this._scheduleManagerStateChanged(); + } + } + + public async closeRunnersAsync(operations?: Operation[]): Promise { + const promises: Promise[] = []; + const recordMap: ReadonlyMap = + this._currentIteration?.records ?? this.resultByOperation; + const closedRecords: Set = new Set(); + for (const operation of operations ?? this.operations) { + if (operation.runner?.closeAsync) { + const record: OperationExecutionRecord | undefined = recordMap.get(operation); + promises.push( + operation.runner.closeAsync().then(() => { + if (record) { + // Collect for batched notification + closedRecords.add(record); + } + }) + ); + } + } + await Promise.all(promises); + if (this.abortController.signal.aborted) { + return; + } + if (closedRecords.size) { + this.hooks.onExecutionStatesUpdated.call(closedRecords); + } + } + + public invalidateOperations(operations?: Iterable, reason?: string): void { + const invalidated: Set = new Set(); + const currentIteration: IExecutionIterationContext | undefined = this._currentIteration; + const currentIterationRecords: Map | undefined = + currentIteration?.records; + for (const operation of operations ?? this.operations) { + const existing: OperationExecutionRecord | undefined = this.resultByOperation.get(operation); + if (existing && TERMINAL_STATUSES.has(existing.status)) { + if (currentIterationRecords?.get(operation) === existing) { + // The record has already executed in the current iteration and was written to + // resultByOperation. Mutating its status now would corrupt the iteration's result + // snapshot (used by the summarizer). Defer the reset until the iteration ends, and + // abort so the operation can be re-run in the next iteration. + this._deferredInvalidations.set(existing, reason); + currentIteration?.abortController.abort(); + } else { + existing.status = OperationStatus.Ready; + invalidated.add(operation); + } + } + } + if (invalidated.size > 0) { + this.hooks.onInvalidateOperations.call(invalidated, reason); + } + if (!currentIteration) { + this._setStatus(OperationStatus.Ready); + } + } + + /** + * Shorthand for scheduling an iteration then executing it. + * Call `abortCurrentIterationAsync()` to cancel the execution of any operations that have not yet begun execution. + * @param iterationOptions - Options for this execution iteration. + * @returns A promise which is resolved when all operations have been processed to a final state. + */ + public async executeAsync(iterationOptions: IOperationGraphIterationOptions): Promise { + await this.abortCurrentIterationAsync(); + const scheduled: IExecutionIterationContext | undefined = + await this._scheduleIterationAsync(iterationOptions); + if (!scheduled) { + return { + operationResults: this.resultByOperation, + status: OperationStatus.NoOp + }; + } + await this.executeScheduledIterationAsync(); + return { + operationResults: scheduled.records, + status: this.status + }; + } + + /** + * Queues a new execution iteration. + * @param iterationOptions - Options for this execution iteration. + * @returns A promise that resolves to true if the iteration was successfully queued, or false if it was not. + */ + public async scheduleIterationAsync(iterationOptions: IOperationGraphIterationOptions): Promise { + return !!(await this._scheduleIterationAsync(iterationOptions)); + } + + /** + * Executes all operations which have been registered, returning a promise which is resolved when all operations have been processed to a final state. + * Aborts the current iteration first, if any. + */ + public async executeScheduledIterationAsync(): Promise { + await this.abortCurrentIterationAsync(); + + const iteration: IExecutionIterationContext | undefined = this._scheduledIteration; + + if (!iteration) { + return false; + } + + this._currentIteration = iteration; + this._setScheduledIteration(undefined); + + iteration.promise = this._executeInnerAsync(this._currentIteration).finally(() => { + this._currentIteration = undefined; + + // Apply any status resets that were deferred because the records were part of the + // now-completed iteration and could not be mutated mid-iteration. + // Coalesce by reason so consumers receive one notification per reason group. + const byReason: Map = new Map(); + for (const [record, deferredReason] of this._deferredInvalidations) { + record.status = OperationStatus.Ready; + let group: Operation[] | undefined = byReason.get(deferredReason); + if (!group) { + group = []; + byReason.set(deferredReason, group); + } + group.push(record.operation); + } + this._deferredInvalidations.clear(); + for (const [deferredReason, ops] of byReason) { + this.hooks.onInvalidateOperations.call(ops, deferredReason); + } + + this._setIdleTimeout(); + }); + + await iteration.promise; + return true; + } + + public async abortCurrentIterationAsync(): Promise { + const iteration: IExecutionIterationContext | undefined = this._currentIteration; + if (iteration) { + iteration.abortController.abort(); + try { + await iteration.promise; + } catch (e) { + // Swallow errors from aborting + } + } + + this._setIdleTimeout(); + } + + public addTerminalDestination(destination: TerminalWritable): void { + this._terminalSplitter.addDestination(destination); + } + + public removeTerminalDestination(destination: TerminalWritable, close: boolean = true): boolean { + return this._terminalSplitter.removeDestination(destination, close); + } + + private _setIdleTimeout(): void { + if (this._currentIteration || this.abortController.signal.aborted) { + return; + } + + if (!this._idleTimeout) { + this._idleTimeout = setTimeout(this._onIdle, 0); + } + } + + private _onIdle = (): void => { + this._idleTimeout = undefined; + if (this._currentIteration || this.abortController.signal.aborted) { + return; + } + + if (!this.pauseNextIteration && this._scheduledIteration) { + void this.executeScheduledIterationAsync(); + } else { + this.hooks.onIdle.call(); + } + }; + + private async _scheduleIterationAsync( + iterationOptions: IOperationGraphIterationOptions + ): Promise { + const { _getInputsSnapshotAsync: getInputsSnapshotAsync } = this; + + const { startTime = performance.now(), inputsSnapshot = await getInputsSnapshotAsync?.() } = + iterationOptions; + const iterationOptionsForCallbacks: IOperationGraphIterationOptions = { startTime, inputsSnapshot }; + + const { hooks } = this; + + const abortController: AbortController = new AbortController(); + + // TERMINAL PIPELINE: + // + // streamCollator --> colorsNewlinesTransform --> StdioWritable + // + const colorsNewlinesTransform: TextRewriterTransform = new TextRewriterTransform({ + destination: this._terminalSplitter, + normalizeNewlines: NewlineKind.OsDefault, + removeColors: !ConsoleTerminalProvider.supportsColor + }); + const terminal: CollatedTerminal = new CollatedTerminal(colorsNewlinesTransform); + const streamCollator: StreamCollator = new StreamCollator({ + destination: colorsNewlinesTransform, + onWriterActive + }); + + const sortedOperations: readonly Operation[] = this._sortedOperations; + + const graph: OperationGraph = this; + + function createEnvironmentForOperation(record: OperationExecutionRecord): IEnvironment { + return hooks.createEnvironmentForOperation.call({ ...process.env }, record); + } + + // Convert the developer graph to the mutable execution graph + const iterationContext: IExecutionIterationContext = { + abortController, + startTime, + streamCollator, + terminal, + inputsSnapshot, + maxParallelism: this._maxParallelism, + onOperationStateChanged: undefined, + createEnvironment: createEnvironmentForOperation, + invalidate: (operations: Iterable, reason: string) => { + graph.invalidateOperations(operations, reason); + }, + get debugMode(): boolean { + return graph.debugMode; + }, + get quietMode(): boolean { + return graph.quietMode; + }, + records: new Map(), + promise: undefined, + completedOperations: 0, + totalOperations: 0 + }; + + const executionRecords: Map = iterationContext.records; + for (const operation of sortedOperations) { + const executionRecord: OperationExecutionRecord = new OperationExecutionRecord( + operation, + iterationContext + ); + + executionRecords.set(operation, executionRecord); + } + + for (const [operation, record] of executionRecords) { + for (const dependency of operation.dependencies) { + const dependencyRecord: OperationExecutionRecord | undefined = executionRecords.get(dependency); + if (!dependencyRecord) { + throw new Error( + `Operation "${record.name}" declares a dependency on operation "${dependency.name}" that is not in the set of operations to execute.` + ); + } + record.dependencies.add(dependencyRecord); + dependencyRecord.consumers.add(record); + } + } + + // Configure operations to execute. + // Ensure we compute the compute the state hashes for all operations before the runtime graph potentially mutates. + if (inputsSnapshot) { + for (const record of executionRecords.values()) { + record.getStateHash(); + } + } + + measureFn(`${PERF_PREFIX}:configureIteration`, () => { + hooks.configureIteration.call(executionRecords, this.resultByOperation, iterationOptionsForCallbacks); + }); + + for (const executionRecord of executionRecords.values()) { + if (!executionRecord.silent) { + // Only count non-silent operations + iterationContext.totalOperations++; + } + } + + if (iterationContext.totalOperations === 0) { + return; + } + + this._setScheduledIteration(iterationContext); + // Notify listeners that an iteration has been scheduled with the planned operation records + try { + this.hooks.onIterationScheduled.call(iterationContext.records); + } catch (e) { + // Surface configuration-time issues clearly + terminal.writeStderrLine( + Colorize.red(`An error occurred in onIterationScheduled hook: ${(e as Error).message}`) + ); + throw e; + } + if (!this._currentIteration) { + this._setIdleTimeout(); + } else if (!this.pauseNextIteration) { + void this.abortCurrentIterationAsync(); + } + return iterationContext; + + function onWriterActive(writer: CollatedWriter | undefined): void { + if (writer) { + iterationContext.completedOperations++; + // Format a header like this + // + // ==[ @rushstack/the-long-thing ]=================[ 1 of 1000 ]== + + // leftPart: "==[ @rushstack/the-long-thing " + const leftPart: string = Colorize.gray('==[') + ' ' + Colorize.cyan(writer.taskName) + ' '; + const leftPartLength: number = 4 + writer.taskName.length + 1; + + // rightPart: " 1 of 1000 ]==" + const completedOfTotal: string = `${iterationContext.completedOperations} of ${iterationContext.totalOperations}`; + const rightPart: string = ' ' + Colorize.white(completedOfTotal) + ' ' + Colorize.gray(']=='); + const rightPartLength: number = 1 + completedOfTotal.length + 4; + + // middlePart: "]=================[" + const twoBracketsLength: number = 2; + const middlePartLengthMinusTwoBrackets: number = Math.max( + ASCII_HEADER_WIDTH - (leftPartLength + rightPartLength + twoBracketsLength), + 0 + ); + + const middlePart: string = Colorize.gray(']' + '='.repeat(middlePartLengthMinusTwoBrackets) + '['); + + terminal.writeStdoutLine('\n' + leftPart + middlePart + rightPart); + + if (!graph.quietMode) { + terminal.writeStdoutLine(''); + } + } + } + } + + /** + * Debounce configuration change notifications so that multiple property setters invoked within the same tick + * only trigger the hook once. This avoids redundant re-computation in listeners (e.g. UI refresh) while preserving + * ordering guarantees that the notification occurs after the initiating state changes are fully applied. + */ + private _scheduleManagerStateChanged(): void { + if (this._graphStateChangeScheduled || this.abortController.signal.aborted) { + return; + } + this._graphStateChangeScheduled = true; + process.nextTick(() => { + this._graphStateChangeScheduled = false; + this.hooks.onGraphStateChanged.call(this); + }); + } + + /** + * Executes all operations which have been registered, returning a promise which is resolved when all operations have been processed to a final state. + * The abortController can be used to cancel the execution of any operations that have not yet begun execution. + */ + private async _executeInnerAsync(iterationContext: IExecutionIterationContext): Promise { + this._setStatus(OperationStatus.Executing); + + const { hooks } = this; + + const { abortController, records: executionRecords, terminal, totalOperations } = iterationContext; + + const isInitial: boolean = this.resultByOperation.size === 0; + + const iterationOptions: IOperationGraphIterationOptions = { + inputsSnapshot: iterationContext.inputsSnapshot, + startTime: iterationContext.startTime + }; + + const executionQueue: AsyncOperationQueue = new AsyncOperationQueue( + executionRecords.values(), + prioritySort + ); + + const abortSignal: AbortSignal = abortController.signal; + + iterationContext.onOperationStateChanged = onOperationStatusChanged; + + // Batched state change tracking using a Set for uniqueness + let batchedStateChanges: Set = new Set(); + function flushBatchedStateChanges(): void { + if (!batchedStateChanges.size) return; + try { + hooks.onExecutionStatesUpdated.call(batchedStateChanges); + } finally { + // Replace the set so that if anything held onto the old one it doesn't get mutated. + batchedStateChanges = new Set(); + } + } + + const state: IStatefulExecutionContext = { + hasAnyFailures: false, + hasAnyNonAllowedWarnings: false, + hasAnyAborted: false, + executionQueue, + resultByOperation: this.resultByOperation, + get completedOperations(): number { + return iterationContext.completedOperations; + }, + set completedOperations(value: number) { + iterationContext.completedOperations = value; + } + }; + + const executionContext: IOperationExecutionContext = { + onStartAsync: onOperationStartAsync, + onResultAsync: onOperationCompleteAsync + }; + + if (!this.quietMode) { + const plural: string = totalOperations === 1 ? '' : 's'; + terminal.writeStdoutLine(`Selected ${totalOperations} operation${plural}:`); + const nonSilentOperations: string[] = []; + for (const record of executionRecords.values()) { + if (!record.silent) { + nonSilentOperations.push(record.name); + } + } + nonSilentOperations.sort(); + for (const name of nonSilentOperations) { + terminal.writeStdoutLine(` ${name}`); + } + terminal.writeStdoutLine(''); + } + + const maxSimultaneousProcesses: number = Math.min(totalOperations, this.parallelism); + // For logging purposes, don't confuse the user by suggesting we might run more operations in parallel than are scheduled. + terminal.writeStdoutLine(`Executing a maximum of ${maxSimultaneousProcesses} simultaneous processes...`); + + const bailStatus: OperationStatus | undefined | void = abortSignal.aborted + ? OperationStatus.Aborted + : await measureAsyncFn( + `${PERF_PREFIX}:beforeExecuteIterationAsync`, + async () => await hooks.beforeExecuteIterationAsync.promise(executionRecords, iterationOptions) + ); + + if (bailStatus) { + // Mark all non-terminal operations as Aborted + for (const record of executionRecords.values()) { + if (!record.isTerminal) { + record.status = OperationStatus.Aborted; + state.hasAnyAborted = true; + } + } + } else { + await measureAsyncFn(`${PERF_PREFIX}:executeOperationsAsync`, async () => { + await Async.forEachAsync( + executionQueue, + async (record: OperationExecutionRecord) => { + if (abortSignal.aborted) { + record.status = OperationStatus.Aborted; + // Bypass the normal completion handler, directly mark the operation as aborted and unblock the queue. + // We do this to ensure that we aren't messing with the stopwatch or terminal. + state.hasAnyAborted = true; + executionQueue.complete(record); + } else { + const lastState: OperationExecutionRecord | undefined = state.resultByOperation.get( + record.operation + ); + await record.executeAsync(lastState, executionContext); + } + }, + { + // In weighted mode, concurrency represents the total "unit budget", not the max number of tasks. + // Do not cap by totalOperations, since that would incorrectly shrink the unit budget and + // reduce parallelism for operations with weight > 1. + concurrency: this.parallelism, + weighted: true, + allowOversubscription: this.allowOversubscription + } + ); + }); + } + + const status: OperationStatus = (() => { + if (bailStatus) return bailStatus; + if (state.hasAnyFailures) return OperationStatus.Failure; + if (state.hasAnyAborted) return OperationStatus.Aborted; + if (state.hasAnyNonAllowedWarnings) return OperationStatus.SuccessWithWarning; + if (iterationContext.totalOperations === 0) return OperationStatus.NoOp; + return OperationStatus.Success; + })(); + + this._setStatus( + (await measureAsyncFn(`${PERF_PREFIX}:afterExecuteIterationAsync`, async () => { + return await hooks.afterExecuteIterationAsync.promise(status, executionRecords, iterationOptions); + })) ?? status + ); + + const { _telemetry: telemetry } = this; + if (telemetry) { + const logEntry: ITelemetryData = measureFn(`${PERF_PREFIX}:prepareTelemetry`, () => { + const isWatch: boolean = this._isWatch; + const jsonOperationResults: Record = {}; + + const durationInSeconds: number = (performance.now() - (iterationContext.startTime ?? 0)) / 1000; + + const extraData: IPhasedExecutionTelemetry = { + ...telemetry.initialExtraData, + isWatch, + // Fields specific to the current operation set + isInitial, + + countAll: 0, + countSuccess: 0, + countSuccessWithWarnings: 0, + countFailure: 0, + countBlocked: 0, + countFromCache: 0, + countSkipped: 0, + countNoOp: 0, + countAborted: 0 + }; + + let changedProjectsOnly: boolean = false; + for (const operation of executionRecords.keys()) { + if (operation.enabled === 'ignore-dependency-changes') { + changedProjectsOnly = true; + break; + } + } + + if (telemetry.changedProjectsOnlyKey) { + // Overwrite this value since we allow changing it at runtime. + extraData[telemetry.changedProjectsOnlyKey] = changedProjectsOnly; + } + + const nonSilentDependenciesByOperation: Map> = new Map(); + function getNonSilentDependencies(operation: Operation): ReadonlySet { + let realDependencies: Set | undefined = nonSilentDependenciesByOperation.get(operation); + if (!realDependencies) { + realDependencies = new Set(); + nonSilentDependenciesByOperation.set(operation, realDependencies); + for (const dependency of operation.dependencies) { + const dependencyRecord: OperationExecutionRecord | undefined = executionRecords.get(dependency); + if (dependencyRecord?.silent) { + for (const deepDependency of getNonSilentDependencies(dependency)) { + realDependencies.add(deepDependency); + } + } else { + realDependencies.add(dependency.name!); + } + } + } + return realDependencies; + } + + for (const [operation, operationResult] of executionRecords) { + if (operationResult.silent) { + // Architectural operation. Ignore. + continue; + } + + const { _operationMetadataManager: operationMetadataManager } = operationResult; + + const { startTime, endTime } = operationResult.stopwatch; + jsonOperationResults[operation.name!] = { + startTimestampMs: startTime, + endTimestampMs: endTime, + nonCachedDurationMs: operationResult.nonCachedDurationMs, + wasExecutedOnThisMachine: operationMetadataManager?.wasCobuilt !== true, + result: operationResult.status, + dependencies: Array.from(getNonSilentDependencies(operation)).sort() + }; + + extraData.countAll++; + switch (operationResult.status) { + case OperationStatus.Success: + extraData.countSuccess++; + break; + case OperationStatus.SuccessWithWarning: + extraData.countSuccessWithWarnings++; + break; + case OperationStatus.Failure: + extraData.countFailure++; + break; + case OperationStatus.Blocked: + extraData.countBlocked++; + break; + case OperationStatus.FromCache: + extraData.countFromCache++; + break; + case OperationStatus.Skipped: + extraData.countSkipped++; + break; + case OperationStatus.NoOp: + extraData.countNoOp++; + break; + case OperationStatus.Aborted: + extraData.countAborted++; + break; + default: + // Do nothing. + break; + } + } + + const innerLogEntry: ITelemetryData = { + name: telemetry.nameForLog, + durationInSeconds, + result: status === OperationStatus.Success ? 'Succeeded' : 'Failed', + extraData, + operationResults: jsonOperationResults + }; + + return innerLogEntry; + }); + + measureFn(`${PERF_PREFIX}:beforeLog`, () => this.hooks.beforeLog.call(logEntry)); + telemetry.log(logEntry); + } + + return status; + + // This function is a callback because it may write to the collatedWriter before + // operation.executeAsync returns (and cleans up the writer) + async function onOperationCompleteAsync(record: OperationExecutionRecord): Promise { + // If the operation is not terminal, we should _only_ notify the queue to assign operations. + if (!record.isTerminal) { + executionQueue.assignOperations(); + } else { + try { + await hooks.afterExecuteOperationAsync.promise(record); + } catch (e) { + _reportOperationErrorIfAny(record); + record.error = e; + record.status = OperationStatus.Failure; + } + _onOperationComplete(record, state); + } + } + + async function onOperationStartAsync( + record: OperationExecutionRecord + ): Promise { + return await hooks.beforeExecuteOperationAsync.promise(record); + } + + function onOperationStatusChanged(record: OperationExecutionRecord): void { + if (record.status === OperationStatus.Ready) { + executionQueue.assignOperations(); + } + const wasEmpty: boolean = batchedStateChanges.size === 0; + batchedStateChanges.add(record); + if (wasEmpty) { + // First change in this microtask; schedule flush + queueMicrotask(flushBatchedStateChanges); + } + } + } +} + +/** + * Handles the result of the operation and propagates any relevant effects. + */ +function _onOperationComplete(record: OperationExecutionRecord, context: IStatefulExecutionContext): void { + const { status } = record; + + switch (status) { + /** + * This operation failed. Mark it as such and all reachable dependents as blocked. + */ + case OperationStatus.Failure: { + _handleOperationFailure(record, context); + break; + } + + /** + * This operation was restored from the build cache. + */ + case OperationStatus.FromCache: { + _handleOperationFromCache(record, context); + break; + } + + /** + * This operation was skipped via legacy change detection. + */ + case OperationStatus.Skipped: { + _handleOperationSkipped(record, context); + break; + } + + /** + * This operation intentionally didn't do anything. + */ + case OperationStatus.NoOp: { + _handleOperationNoOp(record, context); + break; + } + + case OperationStatus.Success: { + _handleOperationSuccess(record, context); + break; + } + + case OperationStatus.SuccessWithWarning: { + _handleOperationSuccessWithWarning(record, context); + break; + } + + case OperationStatus.Aborted: { + _handleOperationAborted(record, context); + break; + } + + default: { + throw new InternalError(`Unexpected operation status: ${status}`); + } + } + + context.executionQueue.complete(record); +} + +/** + * Handle a failed operation and propagate the Blocked status to dependent operations. + */ +function _handleOperationFailure(record: OperationExecutionRecord, context: IStatefulExecutionContext): void { + // Failed operations get reported, even if silent. + // Generally speaking, silent operations shouldn't be able to fail, so this is a safety measure. + _reportOperationErrorIfAny(record); + + const { name } = record; + const { terminal } = record.collatedWriter; // Creates the writer if needed + terminal.writeStderrLine(Colorize.red(`"${name}" failed to build.`)); + + const blockedQueue: Set = new Set(record.consumers); + for (const blockedRecord of blockedQueue) { + if (blockedRecord.status === OperationStatus.Waiting) { + if (!blockedRecord.silent) { + terminal.writeStdoutLine(`"${blockedRecord.name}" is blocked by "${name}".`); + } + blockedRecord.status = OperationStatus.Blocked; + context.executionQueue.complete(blockedRecord); + if (!blockedRecord.silent) { + context.completedOperations++; // Only count non-silent operations + } + for (const dependent of blockedRecord.consumers) { + blockedQueue.add(dependent); + } + } else if (blockedRecord.status !== OperationStatus.Blocked) { + throw new InternalError( + `Blocked operation ${blockedRecord.name} is in an unexpected state: ${blockedRecord.status}` + ); + } + } + context.resultByOperation.set(record.operation, record); + context.hasAnyFailures = true; +} + +/** + * Handle operation restored from cache. + */ +function _handleOperationFromCache( + record: OperationExecutionRecord, + context: IStatefulExecutionContext +): void { + if (!record.silent) { + record.collatedWriter.terminal.writeStdoutLine( + Colorize.green(`"${record.name}" was restored from the build cache.`) + ); + } + context.resultByOperation.set(record.operation, record); +} + +/** + * Handle skipped operation. + */ +function _handleOperationSkipped(record: OperationExecutionRecord, context: IStatefulExecutionContext): void { + // Do not set resultByOperation here. "Skipped" means the operation was not executed, + // so it should not be considered the last *execution* result. + if (!record.silent) { + record.collatedWriter.terminal.writeStdoutLine(Colorize.green(`"${record.name}" was skipped.`)); + } +} + +/** + * Handle no-op operation. + */ +function _handleOperationNoOp(record: OperationExecutionRecord, context: IStatefulExecutionContext): void { + if (!record.silent) { + record.collatedWriter.terminal.writeStdoutLine( + Colorize.gray(`"${record.name}" did not define any work.`) + ); + } + context.resultByOperation.set(record.operation, record); +} + +/** + * Handle successful operation. + */ +function _handleOperationSuccess(record: OperationExecutionRecord, context: IStatefulExecutionContext): void { + const stopwatch: IStopwatchResult = _getOperationStopwatch(record); + if (!record.silent) { + record.collatedWriter.terminal.writeStdoutLine( + Colorize.green(`"${record.name}" completed successfully in ${stopwatch.toString()}.`) + ); + } + context.resultByOperation.set(record.operation, record); +} + +/** + * Handle successful operation with warnings. + */ +function _handleOperationSuccessWithWarning( + record: OperationExecutionRecord, + context: IStatefulExecutionContext +): void { + const stopwatch: IStopwatchResult = _getOperationStopwatch(record); + if (!record.silent) { + record.collatedWriter.terminal.writeStderrLine( + Colorize.yellow(`"${record.name}" completed with warnings in ${stopwatch.toString()}.`) + ); + } + context.resultByOperation.set(record.operation, record); + context.hasAnyNonAllowedWarnings ||= !record.runner.warningsAreAllowed; +} + +/** + * Resolve the appropriate stopwatch for an operation, restoring from metadata if available. + */ +function _getOperationStopwatch(record: OperationExecutionRecord): IStopwatchResult { + const operationMetadataManager: import('./OperationMetadataManager').OperationMetadataManager = + record._operationMetadataManager; + return operationMetadataManager?.tryRestoreStopwatch(record.stopwatch) || record.stopwatch; +} + +/** + * Handle aborted operation. + */ +function _handleOperationAborted(record: OperationExecutionRecord, context: IStatefulExecutionContext): void { + // Do not set resultByOperation here. "Aborted" means the operation was not executed, + // so it should not be considered the last *execution* result. + context.hasAnyAborted = true; +} + +function _reportOperationErrorIfAny(record: OperationExecutionRecord): void { + // Failed operations get reported, even if silent. + // Generally speaking, silent operations shouldn't be able to fail, so this is a safety measure. + let message: string | undefined = undefined; + if (record.error) { + if (!(record.error instanceof AlreadyReportedError)) { + message = record.error.message; + } + } + + if (message) { + // This creates the writer, so don't do this until needed + record.collatedWriter.terminal.writeStderrLine(message); + // Ensure that the summary isn't blank if we have an error message + // If the summary already contains max lines of stderr, this will get dropped, so we hope those lines + // are more useful than the final exit code. + record.stdioSummarizer.writeChunk({ + text: `${message}\n`, + kind: TerminalChunkKind.Stdout + }); + } +} diff --git a/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts b/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts index 4c5af75f6b8..1ff82c3baf6 100644 --- a/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/OperationResultSummarizerPlugin.ts @@ -4,11 +4,7 @@ import { InternalError } from '@rushstack/node-core-library'; import { Colorize, type ITerminal } from '@rushstack/terminal'; -import type { - ICreateOperationsContext, - IPhasedCommandPlugin, - PhasedCommandHooks -} from '../../pluginFramework/PhasedCommandHooks'; +import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import type { IExecutionResult, IOperationExecutionResult } from './IOperationExecutionResult'; import type { Operation } from './Operation'; import { OperationStatus } from './OperationStatus'; @@ -36,12 +32,19 @@ export class OperationResultSummarizerPlugin implements IPhasedCommandPlugin { } public apply(hooks: PhasedCommandHooks): void { - hooks.afterExecuteOperations.tap( - PLUGIN_NAME, - (result: IExecutionResult, context: ICreateOperationsContext): void => { - _printOperationStatus(this._terminal, result); - } - ); + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + // Ensure this plugin runs after all other plugins + graph.hooks.afterExecuteIterationAsync.tap( + PLUGIN_NAME, + ( + status: OperationStatus, + results: ReadonlyMap + ): OperationStatus => { + _printOperationStatus(this._terminal, { status, operationResults: results }); + return status; + } + ); + }); } } @@ -50,10 +53,8 @@ export class OperationResultSummarizerPlugin implements IPhasedCommandPlugin { * @internal */ export function _printOperationStatus(terminal: ITerminal, result: IExecutionResult): void { - const { operationResults } = result; - const operationsByStatus: IOperationsByStatus = new Map(); - for (const record of operationResults) { + for (const record of result.operationResults) { if (record[1].silent) { // Don't report silenced operations continue; diff --git a/libraries/rush-lib/src/logic/operations/OperationStatus.ts b/libraries/rush-lib/src/logic/operations/OperationStatus.ts index 6bff6959ef4..0fe797eda5c 100644 --- a/libraries/rush-lib/src/logic/operations/OperationStatus.ts +++ b/libraries/rush-lib/src/logic/operations/OperationStatus.ts @@ -70,3 +70,12 @@ export const TERMINAL_STATUSES: Set = new Set([ OperationStatus.NoOp, OperationStatus.Aborted ]); + +/** + * The set of statuses that are considered successful and don't trigger a rebuild if current. + */ +export const SUCCESS_STATUSES: Set = new Set([ + OperationStatus.Success, + OperationStatus.FromCache, + OperationStatus.NoOp +]); diff --git a/libraries/rush-lib/src/logic/operations/ParseParallelism.ts b/libraries/rush-lib/src/logic/operations/ParseParallelism.ts new file mode 100644 index 00000000000..915477bf6f2 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/ParseParallelism.ts @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import * as os from 'node:os'; + +import { IS_WINDOWS } from '../../utilities/executionUtilities'; + +let _maxParallelism: number = 0; + +export function getNumberOfCores(): number { + // Ensure this function caches the result (which is expected not to change while the process is loaded), but is expensive to obtain. + return _maxParallelism || (_maxParallelism = os.availableParallelism?.() ?? os.cpus().length); +} + +/** + * A parallelism value expressed as a fraction of total available concurrency slots. + * @beta + */ +export interface IParallelismScalar { + readonly scalar: number; +} + +/** + * A parallelism value, either as an absolute integer count or a scalar fraction of available parallelism. + * @beta + */ +export type Parallelism = number | IParallelismScalar; + +/** + * Since the JSON value is a string, it must be a percentage like "50%", + * which we parse into a scalar in the range (0, 1]. + * The caller is responsible for multiplying by the available parallelism. + */ +export function parseParallelismPercent(weight: string): number { + const percentageRegExp: RegExp = /^\d+(\.\d+)?%$/; + + if (!percentageRegExp.test(weight)) { + throw new Error(`Expecting a percentage string like "12%" or "34.56%".`); + } + + const percentValue: number = parseFloat(weight); + + if (percentValue <= 0) { + throw new Error(`Invalid percentage value of "${percentValue}": value must be greater than zero`); + } + + if (percentValue > 100) { + throw new Error(`Invalid percentage value of "${percentValue}": value must not exceed 100%`); + } + + return percentValue / 100; +} + +/** + * Coerces a `Parallelism` value to a concrete integer number of concurrency units, given the + * maximum number of available slots. + * + * - Raw numeric values are clamped to `[minimum, maxParallelism]`. + * - Scalar values are multiplied by `maxParallelism`, floored, and clamped to `[Math.max(1, minimum), maxParallelism]`. + */ +export function coerceParallelism( + parallelism: Parallelism, + maxParallelism: number, + minimum: number = 0 +): number { + if (typeof parallelism === 'number') { + return Math.max(minimum, Math.min(parallelism, maxParallelism)); + } + // eslint-disable-next-line no-bitwise + return Math.max(Math.max(1, minimum), Math.min((parallelism.scalar * maxParallelism) | 0, maxParallelism)); +} + +/** + * Parses a command line specification for desired parallelism. + * Factored out to enable unit tests + */ +export function parseParallelism(rawParallelism: string | undefined): Parallelism { + if (rawParallelism) { + rawParallelism = rawParallelism.trim(); + + if (rawParallelism === 'max') { + return { scalar: 1 }; + } + + if (rawParallelism.endsWith('%')) { + return { scalar: parseParallelismPercent(rawParallelism) }; + } + + const parallelismAsNumber: number = Number(rawParallelism); + if (!isNaN(parallelismAsNumber)) { + return parallelismAsNumber; + } + + throw new Error( + `Invalid parallelism value of "${rawParallelism}": expected a number, a percentage string, or "max"` + ); + } else { + // If an explicit parallelism number wasn't provided, then choose a sensible + // default. + if (IS_WINDOWS) { + // On desktop Windows, some people have complained that their system becomes + // sluggish if Rush is using all the CPU cores. Leave one thread for + // other operations. For CI environments, you can use the "max" argument to use all available cores. + // Since we use Math.floor when coercing scalars, 0.999 * N = N - 1 for any integer N >= 1. + return { scalar: 0.999 }; + } else { + // Unix-like operating systems have more balanced scheduling, so default + // to the number of CPU cores + return { scalar: 1 }; + } + } +} diff --git a/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts index d83ff4bad07..ac98661fc64 100644 --- a/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/PhasedOperationPlugin.ts @@ -3,13 +3,22 @@ import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { IPhase } from '../../api/CommandLineConfiguration'; -import { Operation } from './Operation'; +import { Operation, type OperationEnabledState } from './Operation'; import type { ICreateOperationsContext, + IOperationGraphContext, IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; +import type { IOperationGraph, IOperationGraphIterationOptions } from './IOperationGraph'; import type { IOperationSettings } from '../../api/RushProjectConfiguration'; +import type { + IConfigurableOperation, + IOperationExecutionResult, + IOperationStateHashComponents +} from './IOperationExecutionResult'; +import { SUCCESS_STATUSES } from './OperationStatus'; +import type { IInputsSnapshot } from '../incremental/InputsSnapshot'; const PLUGIN_NAME: 'PhasedOperationPlugin' = 'PhasedOperationPlugin'; @@ -19,14 +28,14 @@ const PLUGIN_NAME: 'PhasedOperationPlugin' = 'PhasedOperationPlugin'; */ export class PhasedOperationPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { - hooks.createOperations.tap(PLUGIN_NAME, createOperations); + hooks.createOperationsAsync.tap(PLUGIN_NAME, createOperations); // Configure operations later. - hooks.createOperations.tap( + hooks.onGraphCreatedAsync.tap( { name: `${PLUGIN_NAME}.Configure`, stage: 1000 }, - configureOperations + configureExecutionManager ); } } @@ -35,14 +44,27 @@ function createOperations( existingOperations: Set, context: ICreateOperationsContext ): Set { - const { phaseSelection, projectSelection, projectConfigurations } = context; + const { + phaseSelection: phases, + projectSelection: projects, + projectConfigurations, + changedProjectsOnly, + includePhaseDeps, + isIncrementalBuildAllowed, + generateFullGraph, + rushConfiguration + } = context; const operations: Map = new Map(); - // Create tasks for selected phases and projects - // This also creates the minimal set of dependencies needed - for (const phase of phaseSelection) { - for (const project of projectSelection) { + const defaultEnabledState: OperationEnabledState = + changedProjectsOnly && isIncrementalBuildAllowed ? 'ignore-dependency-changes' : true; + + const projectUniverse: Iterable = generateFullGraph + ? rushConfiguration.projects + : projects; + for (const phase of phases) { + for (const project of projectUniverse) { getOrCreateOperation(phase, project); } } @@ -63,11 +85,19 @@ function createOperations( const operationSettings: IOperationSettings | undefined = projectConfigurations .get(project) ?.operationSettingsByOperationName.get(name); + + const includedInSelection: boolean = phases.has(phase) && projects.has(project); operation = new Operation({ project, phase, settings: operationSettings, - logFilenameIdentifier: logFilenameIdentifier + logFilenameIdentifier: logFilenameIdentifier, + enabled: + includePhaseDeps || includedInSelection + ? operationSettings?.ignoreChangedProjectsOnlyFlag + ? true + : defaultEnabledState + : false }); operations.set(key, operation); @@ -93,71 +123,74 @@ function createOperations( } } -function configureOperations(operations: Set, context: ICreateOperationsContext): Set { - const { - changedProjectsOnly, - projectsInUnknownState: changedProjects, - phaseOriginal, - phaseSelection, - projectSelection, - includePhaseDeps, - isInitial - } = context; +function configureExecutionManager(graph: IOperationGraph, context: IOperationGraphContext): void { + graph.hooks.configureIteration.tap( + PLUGIN_NAME, + ( + currentStates: ReadonlyMap, + lastStates: ReadonlyMap, + iterationOptions: IOperationGraphIterationOptions + ) => { + configureOperations(currentStates, lastStates, iterationOptions); + } + ); +} - const basePhases: ReadonlySet = includePhaseDeps ? phaseOriginal : phaseSelection; +function shouldEnableOperation( + currentState: IConfigurableOperation, + lastState: IOperationExecutionResult | undefined, + inputsSnapshot?: IInputsSnapshot +): boolean { + if (!lastState) { + return true; + } - // Grab all operations that were explicitly requested. - const operationsWithWork: Set = new Set(); - for (const operation of operations) { - const { associatedPhase, associatedProject } = operation; - if (basePhases.has(associatedPhase) && changedProjects.has(associatedProject)) { - operationsWithWork.add(operation); - } + if (!SUCCESS_STATUSES.has(lastState.status)) { + return true; } - if (!isInitial && changedProjectsOnly) { - const potentiallyAffectedOperations: Set = new Set(operationsWithWork); - for (const operation of potentiallyAffectedOperations) { - if (operation.settings?.ignoreChangedProjectsOnlyFlag) { - operationsWithWork.add(operation); - } + if (!inputsSnapshot) { + // Insufficient information to tell if a rebuild is needed, so assume yes. + return true; + } - for (const consumer of operation.consumers) { - potentiallyAffectedOperations.add(consumer); - } - } - } else { - // Add all operations that are selected that depend on the explicitly requested operations. - // This will mostly be relevant during watch; in initial runs it should not add any new operations. - for (const operation of operationsWithWork) { - for (const consumer of operation.consumers) { - operationsWithWork.add(consumer); - } - } + const current: IOperationStateHashComponents = currentState.getStateHashComponents(); + const last: IOperationStateHashComponents = lastState.getStateHashComponents(); + + // Always compare local and config hashes + if (current.local !== last.local || current.config !== last.config) { + return true; } - if (includePhaseDeps) { - // Add all operations that are dependencies of the operations already scheduled. - for (const operation of operationsWithWork) { - for (const dependency of operation.dependencies) { - operationsWithWork.add(dependency); - } + const localChangesOnly: boolean = currentState.operation.enabled === 'ignore-dependency-changes'; + if (localChangesOnly) { + return false; + } + + // Compare dependency hashes + if (current.dependencies.length !== last.dependencies.length) { + return true; + } + for (let i: number = 0; i < current.dependencies.length; i++) { + if (current.dependencies[i] !== last.dependencies[i]) { + return true; } } - for (const operation of operations) { - // Enable exactly the set of operations that are requested. - operation.enabled &&= operationsWithWork.has(operation); + return false; +} - if (!includePhaseDeps || !isInitial) { - const { associatedPhase, associatedProject } = operation; +function configureOperations( + currentStates: ReadonlyMap, + lastStates: ReadonlyMap, + iterationOptions: IOperationGraphIterationOptions +): void { + for (const [operation, currentState] of currentStates) { + const lastState: IOperationExecutionResult | undefined = lastStates.get(operation); - // This filter makes the "unsafe" selections happen. - operation.enabled &&= phaseSelection.has(associatedPhase) && projectSelection.has(associatedProject); - } + currentState.enabled = + operation.enabled && shouldEnableOperation(currentState, lastState, iterationOptions.inputsSnapshot); } - - return operations; } // Convert the [IPhase, RushConfigurationProject] into a value suitable for use as a Map key diff --git a/libraries/rush-lib/src/logic/operations/PnpmSyncCopyOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/PnpmSyncCopyOperationPlugin.ts index 457ec654d23..6857acf1ec0 100644 --- a/libraries/rush-lib/src/logic/operations/PnpmSyncCopyOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/PnpmSyncCopyOperationPlugin.ts @@ -22,40 +22,42 @@ export class PnpmSyncCopyOperationPlugin implements IPhasedCommandPlugin { this._terminal = terminal; } public apply(hooks: PhasedCommandHooks): void { - hooks.afterExecuteOperation.tapPromise( - PLUGIN_NAME, - async (runnerContext: IOperationRunnerContext): Promise => { - const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; - const { - status, - operation: { associatedProject: project } - } = record; - - //skip if the phase is skipped or no operation - if ( - status === OperationStatus.Skipped || - status === OperationStatus.NoOp || - status === OperationStatus.Failure - ) { - return; + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.afterExecuteOperationAsync.tapPromise( + PLUGIN_NAME, + async (runnerContext: IOperationRunnerContext): Promise => { + const record: OperationExecutionRecord = runnerContext as OperationExecutionRecord; + const { + status, + operation: { associatedProject: project } + } = record; + + //skip if the phase is skipped or no operation + if ( + status === OperationStatus.Skipped || + status === OperationStatus.NoOp || + status === OperationStatus.Failure + ) { + return; + } + + const pnpmSyncJsonPath: string = `${project.projectFolder}/${RushConstants.nodeModulesFolderName}/${RushConstants.pnpmSyncFilename}`; + if (await FileSystem.exists(pnpmSyncJsonPath)) { + const { PackageExtractor } = await import( + /* webpackChunkName: 'PackageExtractor' */ + '@rushstack/package-extractor' + ); + await pnpmSyncCopyAsync({ + pnpmSyncJsonPath, + ensureFolderAsync: FileSystem.ensureFolderAsync, + forEachAsyncWithConcurrency: Async.forEachAsync, + getPackageIncludedFiles: PackageExtractor.getPackageIncludedFilesAsync, + logMessageCallback: (logMessageOptions: ILogMessageCallbackOptions) => + PnpmSyncUtilities.processLogMessage(logMessageOptions, this._terminal) + }); + } } - - const pnpmSyncJsonPath: string = `${project.projectFolder}/${RushConstants.nodeModulesFolderName}/${RushConstants.pnpmSyncFilename}`; - if (await FileSystem.exists(pnpmSyncJsonPath)) { - const { PackageExtractor } = await import( - /* webpackChunkName: 'PackageExtractor' */ - '@rushstack/package-extractor' - ); - await pnpmSyncCopyAsync({ - pnpmSyncJsonPath, - ensureFolderAsync: FileSystem.ensureFolderAsync, - forEachAsyncWithConcurrency: Async.forEachAsync, - getPackageIncludedFiles: PackageExtractor.getPackageIncludedFilesAsync, - logMessageCallback: (logMessageOptions: ILogMessageCallbackOptions) => - PnpmSyncUtilities.processLogMessage(logMessageOptions, this._terminal) - }); - } - } - ); + ); + }); } } diff --git a/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts index b4f017dfe3f..9be45e9734c 100644 --- a/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShardedPhaseOperationPlugin.ts @@ -39,7 +39,7 @@ const TemplateStringRegexes = { */ export class ShardedPhasedOperationPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { - hooks.createOperations.tap(PLUGIN_NAME, spliceShards); + hooks.createOperationsAsync.tap(PLUGIN_NAME, spliceShards); } } @@ -138,7 +138,8 @@ function spliceShards(existingOperations: Set, context: ICreateOperat project, displayName: collatorDisplayName, rushConfiguration, - commandToRun, + initialCommand: commandToRun, + incrementalCommand: undefined, customParameterValues: collatorParameters, ignoredParameterValues }); @@ -207,7 +208,8 @@ function spliceShards(existingOperations: Set, context: ICreateOperat shardOperation.runner = initializeShellOperationRunner({ phase, project, - commandToRun: baseCommand, + initialCommand: baseCommand, + incrementalCommand: undefined, customParameterValues: shardedParameters, displayName: shardDisplayName, rushConfiguration, diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts index 3f9f5dc6ddb..1ede8388ce2 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunner.ts @@ -10,7 +10,7 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; import { EnvironmentConfiguration } from '../../api/EnvironmentConfiguration'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import { Utilities } from '../../utilities/Utilities'; -import type { IOperationRunner, IOperationRunnerContext } from './IOperationRunner'; +import type { IOperationRunner, IOperationRunnerContext, IOperationLastState } from './IOperationRunner'; import { OperationError } from './OperationError'; import { OperationStatus } from './OperationStatus'; @@ -18,7 +18,8 @@ export interface IShellOperationRunnerOptions { phase: IPhase; rushProject: RushConfigurationProject; displayName: string; - commandToRun: string; + initialCommand: string; + incrementalCommand: string | undefined; commandForHash: string; ignoredParameterValues: ReadonlyArray; } @@ -35,13 +36,14 @@ export class ShellOperationRunner implements IOperationRunner { public readonly silent: boolean = false; public readonly cacheable: boolean = true; public readonly warningsAreAllowed: boolean; - public readonly commandToRun: string; /** * The creator is expected to use a different runner if the command is known to be a noop. */ public readonly isNoOp: boolean = false; private readonly _commandForHash: string; + private readonly _initialCommand: string; + private readonly _incrementalCommand: string | undefined; private readonly _rushProject: RushConfigurationProject; @@ -54,24 +56,16 @@ export class ShellOperationRunner implements IOperationRunner { this.warningsAreAllowed = EnvironmentConfiguration.allowWarningsInSuccessfulBuild || phase.allowWarningsOnSuccess || false; this._rushProject = options.rushProject; - this.commandToRun = options.commandToRun; + this._initialCommand = options.initialCommand; + this._incrementalCommand = options.incrementalCommand; this._commandForHash = options.commandForHash; this._ignoredParameterValues = options.ignoredParameterValues; } - public async executeAsync(context: IOperationRunnerContext): Promise { - try { - return await this._executeAsync(context); - } catch (error) { - throw new OperationError('executing', (error as Error).message); - } - } - - public getConfigHash(): string { - return this._commandForHash; - } - - private async _executeAsync(context: IOperationRunnerContext): Promise { + public async executeAsync( + context: IOperationRunnerContext, + lastState?: IOperationLastState + ): Promise { return await context.runWithTerminalAsync( async (terminal: ITerminal, terminalProvider: ITerminalProvider) => { let hasWarningOrError: boolean = false; @@ -82,27 +76,25 @@ export class ShellOperationRunner implements IOperationRunner { `These parameters were ignored for this operation by project-level configuration: ${this._ignoredParameterValues.join(' ')}` ); } + const commandToRun: string = (lastState && this._incrementalCommand) || this._initialCommand; // Run the operation - terminal.writeLine(`Invoking: ${this.commandToRun}`); + terminal.writeLine(`Invoking (${lastState ? 'incremental' : 'initial'}): ${commandToRun}`); const { rushConfiguration, projectFolder } = this._rushProject; const { environment: initialEnvironment } = context; - const subProcess: child_process.ChildProcess = Utilities.executeLifecycleCommandAsync( - this.commandToRun, - { - rushConfiguration: rushConfiguration, - workingDirectory: projectFolder, - initCwd: rushConfiguration.commonTempFolder, - handleOutput: true, - environmentPathOptions: { - includeProjectBin: true - }, - initialEnvironment - } - ); + const subProcess: child_process.ChildProcess = Utilities.executeLifecycleCommandAsync(commandToRun, { + rushConfiguration: rushConfiguration, + workingDirectory: projectFolder, + initCwd: rushConfiguration.commonTempFolder, + handleOutput: true, + environmentPathOptions: { + includeProjectBin: true + }, + initialEnvironment + }); // Hook into events, in order to get live streaming of the log subProcess.stdout?.on('data', (data: Buffer) => { @@ -146,6 +138,10 @@ export class ShellOperationRunner implements IOperationRunner { } ); } + + public getConfigHash(): string { + return this._commandForHash; + } } /** diff --git a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts index 8e8edca5340..990aedb6177 100644 --- a/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ShellOperationRunnerPlugin.ts @@ -24,13 +24,13 @@ export const PLUGIN_NAME: 'ShellOperationRunnerPlugin' = 'ShellOperationRunnerPl */ export class ShellOperationRunnerPlugin implements IPhasedCommandPlugin { public apply(hooks: PhasedCommandHooks): void { - hooks.createOperations.tap( + hooks.createOperationsAsync.tap( PLUGIN_NAME, function createShellOperations( operations: Set, context: ICreateOperationsContext ): Set { - const { rushConfiguration, isInitial } = context; + const { rushConfiguration, isIncrementalBuildAllowed } = context; const getCustomParameterValues: (operation: Operation) => ICustomParameterValuesForOperation = getCustomParameterValuesByOperation(); @@ -52,19 +52,20 @@ export class ShellOperationRunnerPlugin implements IPhasedCommandPlugin { // This is the command that will be used to identify the cache entry for this operation const commandForHash: string | undefined = shellCommand ?? scripts?.[phaseName]; - // For execution of non-initial runs, prefer the `:incremental` script if it exists. + // For execution of non-initial iterations, prefer the `:incremental` script if it exists. // However, the `shellCommand` value still takes precedence per the spec for that feature. - const commandToRun: string | undefined = - shellCommand ?? - (!isInitial ? scripts?.[`${phaseName}:incremental`] : undefined) ?? - scripts?.[phaseName]; + const initialCommand: string | undefined = shellCommand ?? scripts?.[phaseName]; + const incrementalCommand: string | undefined = isIncrementalBuildAllowed + ? (shellCommand ?? scripts?.[`${phaseName}:incremental`]) + : undefined; operation.runner = initializeShellOperationRunner({ phase, project, displayName, commandForHash, - commandToRun, + initialCommand, + incrementalCommand, customParameterValues, ignoredParameterValues, rushConfiguration @@ -83,29 +84,41 @@ export function initializeShellOperationRunner(options: { project: RushConfigurationProject; displayName: string; rushConfiguration: RushConfiguration; - commandToRun: string | undefined; + initialCommand: string | undefined; + incrementalCommand: string | undefined; commandForHash?: string; customParameterValues: ReadonlyArray; ignoredParameterValues: ReadonlyArray; }): IOperationRunner { - const { phase, project, commandToRun: rawCommandToRun, displayName, ignoredParameterValues } = options; - - if (typeof rawCommandToRun !== 'string' && phase.missingScriptBehavior === 'error') { + const { + phase, + project, + initialCommand: rawInitialCommand, + incrementalCommand: rawIncrementalCommand, + displayName, + ignoredParameterValues + } = options; + + if (typeof rawInitialCommand !== 'string' && phase.missingScriptBehavior === 'error') { throw new Error( `The project '${project.packageName}' does not define a '${phase.name}' command in the 'scripts' section of its package.json` ); } - if (rawCommandToRun) { + if (rawInitialCommand) { const { commandForHash: rawCommandForHash, customParameterValues } = options; - const commandToRun: string = formatCommand(rawCommandToRun, customParameterValues); + const initialCommand: string = formatCommand(rawInitialCommand, customParameterValues); + const incrementalCommand: string | undefined = rawIncrementalCommand + ? formatCommand(rawIncrementalCommand, customParameterValues) + : undefined; const commandForHash: string = rawCommandForHash ? formatCommand(rawCommandForHash, customParameterValues) - : commandToRun; + : initialCommand; return new ShellOperationRunner({ - commandToRun, + initialCommand, + incrementalCommand, commandForHash, displayName, phase, diff --git a/libraries/rush-lib/src/logic/operations/ValidateOperationsPlugin.ts b/libraries/rush-lib/src/logic/operations/ValidateOperationsPlugin.ts index 3b75af31dd2..0f256c4789b 100644 --- a/libraries/rush-lib/src/logic/operations/ValidateOperationsPlugin.ts +++ b/libraries/rush-lib/src/logic/operations/ValidateOperationsPlugin.ts @@ -3,13 +3,7 @@ import type { ITerminal } from '@rushstack/terminal'; -import type { Operation } from './Operation'; -import type { - ICreateOperationsContext, - IPhasedCommandPlugin, - PhasedCommandHooks -} from '../../pluginFramework/PhasedCommandHooks'; -import type { IOperationExecutionResult } from './IOperationExecutionResult'; +import type { IPhasedCommandPlugin, PhasedCommandHooks } from '../../pluginFramework/PhasedCommandHooks'; import type { RushConfigurationProject } from '../../api/RushConfigurationProject'; import type { RushProjectConfiguration } from '../../api/RushProjectConfiguration'; import type { IPhase } from '../../api/CommandLineConfiguration'; @@ -17,8 +11,7 @@ import type { IPhase } from '../../api/CommandLineConfiguration'; const PLUGIN_NAME: 'ValidateOperationsPlugin' = 'ValidateOperationsPlugin'; /** - * Core phased command plugin that provides the functionality for generating a base operation graph - * from the set of selected projects and phases. + * Core phased command plugin that verifies correctness of the entries in rush-project.json */ export class ValidateOperationsPlugin implements IPhasedCommandPlugin { private readonly _terminal: ITerminal; @@ -28,34 +21,29 @@ export class ValidateOperationsPlugin implements IPhasedCommandPlugin { } public apply(hooks: PhasedCommandHooks): void { - hooks.beforeExecuteOperations.tap(PLUGIN_NAME, this._validateOperations.bind(this)); - } - - private _validateOperations( - records: Map, - context: ICreateOperationsContext - ): void { - const phasesByProject: Map> = new Map(); - for (const { associatedPhase, associatedProject, runner } of records.keys()) { - if (!runner?.isNoOp) { - // Ignore operations that aren't associated with a project or phase, or that - // use the NullOperationRunner (i.e. - the phase doesn't do anything) - let projectPhases: Set | undefined = phasesByProject.get(associatedProject); - if (!projectPhases) { - projectPhases = new Set(); - phasesByProject.set(associatedProject, projectPhases); + hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph, context) => { + const phasesByProject: Map> = new Map(); + for (const { associatedPhase, associatedProject, runner } of graph.operations) { + if (!runner?.isNoOp) { + // Ignore operations that aren't associated with a project or phase, or that + // use the NullOperationRunner (i.e. - the phase doesn't do anything) + let projectPhases: Set | undefined = phasesByProject.get(associatedProject); + if (!projectPhases) { + projectPhases = new Set(); + phasesByProject.set(associatedProject, projectPhases); + } + + projectPhases.add(associatedPhase); } - - projectPhases.add(associatedPhase); } - } - for (const [project, phases] of phasesByProject) { - const projectConfiguration: RushProjectConfiguration | undefined = - context.projectConfigurations.get(project); - if (projectConfiguration) { - projectConfiguration.validatePhaseConfiguration(phases, this._terminal); + for (const [project, phases] of phasesByProject) { + const projectConfiguration: RushProjectConfiguration | undefined = + context.projectConfigurations.get(project); + if (projectConfiguration) { + projectConfiguration.validatePhaseConfiguration(phases, this._terminal); + } } - } + }); } } diff --git a/libraries/rush-lib/src/logic/operations/WeightedOperationPlugin.ts b/libraries/rush-lib/src/logic/operations/WeightedOperationPlugin.ts deleted file mode 100644 index ae77b9be00e..00000000000 --- a/libraries/rush-lib/src/logic/operations/WeightedOperationPlugin.ts +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import { Async } from '@rushstack/node-core-library'; - -import type { Operation } from './Operation'; -import type { - ICreateOperationsContext, - IPhasedCommandPlugin, - PhasedCommandHooks -} from '../../pluginFramework/PhasedCommandHooks'; -import type { IOperationSettings, RushProjectConfiguration } from '../../api/RushProjectConfiguration'; -import type { IOperationExecutionResult } from './IOperationExecutionResult'; -import type { OperationExecutionRecord } from './OperationExecutionRecord'; -import { parseParallelismPercent } from '../../cli/parsing/ParseParallelism'; - -const PLUGIN_NAME: 'WeightedOperationPlugin' = 'WeightedOperationPlugin'; - -/** - * Add weights to operations based on the operation settings in rush-project.json. - * - * This also sets the weight of no-op operations to 0. - */ -export class WeightedOperationPlugin implements IPhasedCommandPlugin { - public apply(hooks: PhasedCommandHooks): void { - hooks.beforeExecuteOperations.tap(PLUGIN_NAME, weightOperations); - } -} - -function weightOperations( - operations: Map, - context: ICreateOperationsContext -): Map { - const { projectConfigurations } = context; - - for (const [operation, record] of operations) { - const { runner } = record as OperationExecutionRecord; - const { associatedProject: project, associatedPhase: phase } = operation; - if (runner!.isNoOp) { - operation.weight = 0; - } else { - const projectConfiguration: RushProjectConfiguration | undefined = projectConfigurations.get(project); - const operationSettings: IOperationSettings | undefined = - operation.settings ?? projectConfiguration?.operationSettingsByOperationName.get(phase.name); - if (operationSettings?.weight !== undefined) { - if (typeof operationSettings.weight === 'number') { - operation.weight = operationSettings.weight; - } else if (typeof operationSettings.weight === 'string') { - try { - operation.weight = parseParallelismPercent(operationSettings.weight); - } catch (error) { - throw new Error( - `${operation.name} (invalid weight: ${JSON.stringify(operationSettings.weight)}) ${(error as Error).message}` - ); - } - } - } - } - Async.validateWeightedIterable(operation); - } - return operations; -} diff --git a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts index fcd1a87a8d4..6acf633c4e8 100644 --- a/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/AsyncOperationQueue.test.ts @@ -51,7 +51,7 @@ function createRecord(name: string): OperationExecutionRecord { phase: mockPhase, project: getOrCreateProject(name) }), - {} as unknown as IOperationExecutionRecordContext + { maxParallelism: 10 } as unknown as IOperationExecutionRecordContext ); } diff --git a/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts index c5ce91c6fd0..daad7a03431 100644 --- a/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/BuildPlanPlugin.test.ts @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. +import path from 'node:path'; import { MockWritable, StringBufferTerminalProvider, Terminal } from '@rushstack/terminal'; import { JsonFile } from '@rushstack/node-core-library'; -import { StreamCollator } from '@rushstack/stream-collator'; import { BuildPlanPlugin } from '../BuildPlanPlugin'; import { type ICreateOperationsContext, - type IExecuteOperationsContext, + type IOperationGraphContext as IOperationExecutionManagerContext, PhasedCommandHooks } from '../../../pluginFramework/PhasedCommandHooks'; import type { Operation } from '../Operation'; @@ -17,14 +17,13 @@ import { type IPhase, type IPhasedCommandConfig } from '../../../api/CommandLineConfiguration'; -import { OperationExecutionRecord } from '../OperationExecutionRecord'; import { PhasedOperationPlugin } from '../PhasedOperationPlugin'; import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; import { RushConstants } from '../../RushConstants'; import { MockOperationRunner } from './MockOperationRunner'; -import path from 'node:path'; import type { ICommandLineJson } from '../../../api/CommandLineJson'; import type { IInputsSnapshot } from '../../incremental/InputsSnapshot'; +import { OperationGraph } from '../OperationGraph'; describe(BuildPlanPlugin.name, () => { const rushJsonFile: string = path.resolve(__dirname, `../../test/workspaceRepo/rush.json`); @@ -37,9 +36,6 @@ describe(BuildPlanPlugin.name, () => { let stringBufferTerminalProvider!: StringBufferTerminalProvider; let terminal!: Terminal; const mockStreamWritable: MockWritable = new MockWritable(); - const streamCollator = new StreamCollator({ - destination: mockStreamWritable - }); beforeEach(() => { stringBufferTerminalProvider = new StringBufferTerminalProvider(); terminal = new Terminal(stringBufferTerminalProvider); @@ -67,76 +63,82 @@ describe(BuildPlanPlugin.name, () => { } async function testCreateOperationsAsync( + hooks: PhasedCommandHooks, phaseSelection: Set, projectSelection: Set, changedProjects: Set - ): Promise> { - const hooks: PhasedCommandHooks = new PhasedCommandHooks(); - // Apply the plugin being tested - new PhasedOperationPlugin().apply(hooks); + ): Promise { // Add mock runners for included operations. - hooks.createOperations.tap('MockOperationRunnerPlugin', createMockRunner); + hooks.createOperationsAsync.tap('MockOperationRunnerPlugin', createMockRunner); - const context: Pick< + const createOperationsContext: Pick< ICreateOperationsContext, - | 'phaseOriginal' - | 'phaseSelection' - | 'projectSelection' - | 'projectsInUnknownState' - | 'projectConfigurations' + 'phaseSelection' | 'projectSelection' | 'projectConfigurations' > = { - phaseOriginal: phaseSelection, phaseSelection, projectSelection, - projectsInUnknownState: changedProjects, projectConfigurations: new Map() }; - const operations: Set = await hooks.createOperations.promise( + const operations: Set = await hooks.createOperationsAsync.promise( new Set(), - context as ICreateOperationsContext + createOperationsContext as ICreateOperationsContext ); - return operations; + const graph: OperationGraph = new OperationGraph(operations, { + debugMode: false, + quietMode: true, + destinations: [mockStreamWritable], + allowOversubscription: true, + parallelism: 1, + abortController: new AbortController() + }); + + const operationManagerContext: Pick< + IOperationExecutionManagerContext, + 'projectConfigurations' | 'phaseSelection' | 'projectSelection' + > = { + projectConfigurations: new Map(), + phaseSelection, + projectSelection + }; + + await hooks.onGraphCreatedAsync.promise( + graph, + operationManagerContext as IOperationExecutionManagerContext + ); + + return graph; } describe('build plan debugging', () => { it('should generate a build plan', async () => { const hooks: PhasedCommandHooks = new PhasedCommandHooks(); - + new PhasedOperationPlugin().apply(hooks); + // Apply the plugin being tested new BuildPlanPlugin(terminal).apply(hooks); - const inputsSnapshot: Pick = { - getTrackedFileHashesForOperation() { - return new Map(); - } - }; - const context: Pick = { - inputsSnapshot: inputsSnapshot as unknown as IInputsSnapshot, - projectConfigurations: new Map() - }; const buildCommand: IPhasedCommandConfig = commandLineConfiguration.commands.get( 'build' )! as IPhasedCommandConfig; - const operationMap = new Map(); - - const operations = await testCreateOperationsAsync( + const graph = await testCreateOperationsAsync( + hooks, buildCommand.phases, new Set(rushConfiguration.projects), new Set(rushConfiguration.projects) ); - operations.forEach((operation) => { - operationMap.set( - operation, - new OperationExecutionRecord(operation, { - debugMode: false, - quietMode: true, - streamCollator, - inputsSnapshot: undefined - }) - ); - }); - - await hooks.beforeExecuteOperations.promise(operationMap, context as IExecuteOperationsContext); + + const inputsSnapshot: Pick< + IInputsSnapshot, + 'getTrackedFileHashesForOperation' | 'getOperationOwnStateHash' + > = { + getTrackedFileHashesForOperation() { + return new Map(); + }, + getOperationOwnStateHash() { + return '0'; + } + }; + await graph.executeAsync({ inputsSnapshot: inputsSnapshot as IInputsSnapshot }); expect( stringBufferTerminalProvider.getAllOutputAsChunks({ diff --git a/libraries/rush-lib/src/logic/operations/test/IgnoredParametersPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/IgnoredParametersPlugin.test.ts index ee6b3fe6d3b..de074eff375 100644 --- a/libraries/rush-lib/src/logic/operations/test/IgnoredParametersPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/IgnoredParametersPlugin.test.ts @@ -17,8 +17,11 @@ import { } from '../IgnoredParametersPlugin'; import { type ICreateOperationsContext, + type IOperationGraphContext, PhasedCommandHooks } from '../../../pluginFramework/PhasedCommandHooks'; +import type { IOperationGraph } from '../IOperationGraph'; +import { OperationGraphHooks } from '../../../pluginFramework/OperationGraphHooks'; import { RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; import type { IEnvironment } from '../../../utilities/Utilities'; import type { IOperationRunnerContext } from '../IOperationRunner'; @@ -61,17 +64,10 @@ describe(IgnoredParametersPlugin.name, () => { const fakeCreateOperationsContext: Pick< ICreateOperationsContext, - | 'phaseOriginal' - | 'phaseSelection' - | 'projectSelection' - | 'projectsInUnknownState' - | 'projectConfigurations' - | 'rushConfiguration' + 'phaseSelection' | 'projectSelection' | 'projectConfigurations' | 'rushConfiguration' > = { - phaseOriginal: buildCommand.phases, phaseSelection: buildCommand.phases, projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set(rushConfiguration.projects), projectConfigurations, rushConfiguration }; @@ -83,9 +79,17 @@ describe(IgnoredParametersPlugin.name, () => { new ShellOperationRunnerPlugin().apply(hooks); new IgnoredParametersPlugin().apply(hooks); - const operations: Set = await hooks.createOperations.promise( + const operations: Set = await hooks.createOperationsAsync.promise( new Set(), - fakeCreateOperationsContext as ICreateOperationsContext + fakeCreateOperationsContext as unknown as ICreateOperationsContext + ); + + // Set up a mock graph and invoke onGraphCreatedAsync so the plugin registers its graph hooks + const graphHooks: OperationGraphHooks = new OperationGraphHooks(); + const fakeGraph: IOperationGraph = { hooks: graphHooks } as unknown as IOperationGraph; + await hooks.onGraphCreatedAsync.promise( + fakeGraph, + fakeCreateOperationsContext as unknown as IOperationGraphContext ); // Test project 'a' which has parameterNamesToIgnore: ["--production"] @@ -96,7 +100,7 @@ describe(IgnoredParametersPlugin.name, () => { const mockRecordA = createMockRecord(operationA!); // Call the hook to get the environment - const envA: IEnvironment = hooks.createEnvironmentForOperation.call({ ...process.env }, mockRecordA); + const envA: IEnvironment = graphHooks.createEnvironmentForOperation.call({ ...process.env }, mockRecordA); // Verify the environment variable is set correctly for project 'a' expect(envA[RUSHSTACK_CLI_IGNORED_PARAMETER_NAMES_ENV_VAR]).toBe('["--production"]'); @@ -107,7 +111,7 @@ describe(IgnoredParametersPlugin.name, () => { const mockRecordB = createMockRecord(operationB!); - const envB: IEnvironment = hooks.createEnvironmentForOperation.call({ ...process.env }, mockRecordB); + const envB: IEnvironment = graphHooks.createEnvironmentForOperation.call({ ...process.env }, mockRecordB); // Verify the environment variable is set correctly for project 'b' expect(envB[RUSHSTACK_CLI_IGNORED_PARAMETER_NAMES_ENV_VAR]).toBe( @@ -132,17 +136,10 @@ describe(IgnoredParametersPlugin.name, () => { const fakeCreateOperationsContext: Pick< ICreateOperationsContext, - | 'phaseOriginal' - | 'phaseSelection' - | 'projectSelection' - | 'projectsInUnknownState' - | 'projectConfigurations' - | 'rushConfiguration' + 'phaseSelection' | 'projectSelection' | 'projectConfigurations' | 'rushConfiguration' > = { - phaseOriginal: echoCommand.phases, phaseSelection: echoCommand.phases, projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set(rushConfiguration.projects), projectConfigurations: new Map(), rushConfiguration }; @@ -154,9 +151,17 @@ describe(IgnoredParametersPlugin.name, () => { new ShellOperationRunnerPlugin().apply(hooks); new IgnoredParametersPlugin().apply(hooks); - const operations: Set = await hooks.createOperations.promise( + const operations: Set = await hooks.createOperationsAsync.promise( new Set(), - fakeCreateOperationsContext as ICreateOperationsContext + fakeCreateOperationsContext as unknown as ICreateOperationsContext + ); + + // Set up a mock graph and invoke onGraphCreatedAsync so the plugin registers its graph hooks + const graphHooks: OperationGraphHooks = new OperationGraphHooks(); + const fakeGraph: IOperationGraph = { hooks: graphHooks } as unknown as IOperationGraph; + await hooks.onGraphCreatedAsync.promise( + fakeGraph, + fakeCreateOperationsContext as unknown as IOperationGraphContext ); // Get any operation @@ -165,7 +170,7 @@ describe(IgnoredParametersPlugin.name, () => { const mockRecord = createMockRecord(operation); - const env: IEnvironment = hooks.createEnvironmentForOperation.call({ ...process.env }, mockRecord); + const env: IEnvironment = graphHooks.createEnvironmentForOperation.call({ ...process.env }, mockRecord); // Verify the environment variable is not set expect(env[RUSHSTACK_CLI_IGNORED_PARAMETER_NAMES_ENV_VAR]).toBeUndefined(); diff --git a/libraries/rush-lib/src/logic/operations/test/Operation.test.ts b/libraries/rush-lib/src/logic/operations/test/Operation.test.ts new file mode 100644 index 00000000000..7428bf14478 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/test/Operation.test.ts @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { IPhase } from '../../../api/CommandLineConfiguration'; +import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; +import type { IOperationSettings } from '../../../api/RushProjectConfiguration'; +import { Operation } from '../Operation'; +import { MockOperationRunner } from './MockOperationRunner'; + +const MOCK_PHASE: IPhase = { + name: '_phase:test', + allowWarningsOnSuccess: false, + associatedParameters: new Set(), + dependencies: { + self: new Set(), + upstream: new Set() + }, + isSynthetic: false, + logFilenameIdentifier: '_phase_test', + missingScriptBehavior: 'silent' +}; + +function createProject(packageName: string): RushConfigurationProject { + return { + packageName + } as RushConfigurationProject; +} + +function createOperation(options: { + project: RushConfigurationProject; + settings?: IOperationSettings; + isNoOp?: boolean; +}): Operation { + const { project, settings, isNoOp } = options; + return new Operation({ + phase: MOCK_PHASE, + project, + settings, + runner: new MockOperationRunner(`${project.packageName} (${MOCK_PHASE.name})`, undefined, false, isNoOp), + logFilenameIdentifier: `${project.packageName}_phase_test` + }); +} + +describe('Operation weight assignment', () => { + it('applies numeric weight from operation settings', () => { + const project: RushConfigurationProject = createProject('project-number'); + const operation: Operation = createOperation({ + project, + settings: { + operationName: MOCK_PHASE.name, + weight: 7 + } + }); + + expect(operation.weight).toBe(7); + }); + + it('parses percentage weight as a scalar', () => { + const project: RushConfigurationProject = createProject('project-percent'); + const operation: Operation = createOperation({ + project, + settings: { + operationName: MOCK_PHASE.name, + weight: '25%' + } as IOperationSettings + }); + + expect(operation.weight).toEqual({ scalar: 0.25 }); + }); + + it('parses 50% weight as scalar 0.5', () => { + const project: RushConfigurationProject = createProject('project-config'); + const operation: Operation = createOperation({ + project, + settings: { + operationName: MOCK_PHASE.name, + weight: '50%' + } as IOperationSettings + }); + + expect(operation.weight).toEqual({ scalar: 0.5 }); + }); + + it('parses fractional percentage weight as a scalar', () => { + const project: RushConfigurationProject = createProject('project-floor'); + const operation: Operation = createOperation({ + project, + settings: { + operationName: MOCK_PHASE.name, + weight: '33.3333%' + } as IOperationSettings + }); + + expect(operation.weight).toEqual({ scalar: 0.333333 }); + }); + + it('throws for invalid percentage weight format', () => { + const project: RushConfigurationProject = createProject('project-invalid'); + expect(() => { + createOperation({ + project, + // @ts-expect-error Testing invalid input + settings: { + operationName: MOCK_PHASE.name, + weight: '12.5a%' + } as IOperationSettings + }); + }).toThrow(/invalid weight for operation/i); + }); +}); diff --git a/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts b/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts deleted file mode 100644 index c74b2a9a3d9..00000000000 --- a/libraries/rush-lib/src/logic/operations/test/OperationExecutionManager.test.ts +++ /dev/null @@ -1,716 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -// The TaskExecutionManager prints "x.xx seconds" in TestRunner.test.ts.snap; ensure that the Stopwatch timing is deterministic -jest.mock('../../../utilities/Utilities'); -jest.mock('../OperationStateFile'); - -jest.mock('@rushstack/terminal', () => { - const originalModule = jest.requireActual('@rushstack/terminal'); - return { - ...originalModule, - ConsoleTerminalProvider: { - ...originalModule.ConsoleTerminalProvider, - supportsColor: true - } - }; -}); - -import { Terminal, MockWritable, PrintUtilities } from '@rushstack/terminal'; -import { CollatedTerminal } from '@rushstack/stream-collator'; -import { Async } from '@rushstack/node-core-library'; - -import type { IPhase } from '../../../api/CommandLineConfiguration'; -import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; -import { - OperationExecutionManager, - type IOperationExecutionManagerOptions -} from '../OperationExecutionManager'; -import { _printOperationStatus } from '../OperationResultSummarizerPlugin'; -import { _printTimeline } from '../ConsoleTimelinePlugin'; -import { OperationStatus } from '../OperationStatus'; -import { Operation } from '../Operation'; -import { Utilities } from '../../../utilities/Utilities'; -import type { IOperationRunner } from '../IOperationRunner'; -import { MockOperationRunner } from './MockOperationRunner'; -import type { IExecutionResult, IOperationExecutionResult } from '../IOperationExecutionResult'; -import { CollatedTerminalProvider } from '../../../utilities/CollatedTerminalProvider'; -import type { CobuildConfiguration } from '../../../api/CobuildConfiguration'; -import type { OperationStateFile } from '../OperationStateFile'; - -const mockGetTimeInMs: jest.Mock = jest.fn(); -Utilities.getTimeInMs = mockGetTimeInMs; - -let mockTimeInMs: number = 0; -mockGetTimeInMs.mockImplementation(() => { - mockTimeInMs += 100; - return mockTimeInMs; -}); - -const mockWritable: MockWritable = new MockWritable(); -const mockTerminal: Terminal = new Terminal(new CollatedTerminalProvider(new CollatedTerminal(mockWritable))); - -const mockPhase: IPhase = { - name: 'phase', - allowWarningsOnSuccess: false, - associatedParameters: new Set(), - dependencies: { - self: new Set(), - upstream: new Set() - }, - isSynthetic: false, - logFilenameIdentifier: 'phase', - missingScriptBehavior: 'silent' -}; -const projectsByName: Map = new Map(); -function getOrCreateProject(name: string): RushConfigurationProject { - let project: RushConfigurationProject | undefined = projectsByName.get(name); - if (!project) { - project = { - packageName: name - } as unknown as RushConfigurationProject; - projectsByName.set(name, project); - } - return project; -} - -function createExecutionManager( - executionManagerOptions: IOperationExecutionManagerOptions, - operationRunner: IOperationRunner -): OperationExecutionManager { - const operation: Operation = new Operation({ - runner: operationRunner, - logFilenameIdentifier: 'operation', - phase: mockPhase, - project: getOrCreateProject('project') - }); - - return new OperationExecutionManager(new Set([operation]), executionManagerOptions); -} - -describe(OperationExecutionManager.name, () => { - let executionManager: OperationExecutionManager; - let executionManagerOptions: IOperationExecutionManagerOptions; - - beforeEach(() => { - jest.spyOn(PrintUtilities, 'getConsoleWidth').mockReturnValue(90); - mockWritable.reset(); - }); - - describe('Error logging', () => { - beforeEach(() => { - executionManagerOptions = { - quietMode: false, - debugMode: false, - parallelism: 1, - allowOversubscription: true, - destination: mockWritable - }; - }); - - it('printedStderrAfterError', async () => { - executionManager = createExecutionManager( - executionManagerOptions, - new MockOperationRunner('stdout+stderr', async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1\n'); - terminal.writeStderrLine('Error: step 1 failed\n'); - return OperationStatus.Failure; - }) - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await executionManager.executeAsync(abortController); - _printOperationStatus(mockTerminal, result); - expect(result.status).toEqual(OperationStatus.Failure); - expect(result.operationResults.size).toEqual(1); - const firstResult: IOperationExecutionResult | undefined = result.operationResults - .values() - .next().value; - expect(firstResult?.status).toEqual(OperationStatus.Failure); - - const allMessages: string = mockWritable.getAllOutput(); - expect(allMessages).toContain('Error: step 1 failed'); - expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); - }); - - it('printedStdoutAfterErrorWithEmptyStderr', async () => { - executionManager = createExecutionManager( - executionManagerOptions, - new MockOperationRunner('stdout only', async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1\n'); - terminal.writeStdoutLine('Error: step 1 failed\n'); - return OperationStatus.Failure; - }) - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await executionManager.executeAsync(abortController); - _printOperationStatus(mockTerminal, result); - expect(result.status).toEqual(OperationStatus.Failure); - expect(result.operationResults.size).toEqual(1); - const firstResult: IOperationExecutionResult | undefined = result.operationResults - .values() - .next().value; - expect(firstResult?.status).toEqual(OperationStatus.Failure); - - const allOutput: string = mockWritable.getAllOutput(); - expect(allOutput).toMatch(/Build step 1/); - expect(allOutput).toMatch(/Error: step 1 failed/); - expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); - }); - }); - - describe('Aborting', () => { - it('Aborted operations abort', async () => { - const mockRun: jest.Mock = jest.fn(); - - const firstOperation = new Operation({ - runner: new MockOperationRunner('1', mockRun), - phase: mockPhase, - project: getOrCreateProject('1'), - logFilenameIdentifier: '1' - }); - - const secondOperation = new Operation({ - runner: new MockOperationRunner('2', mockRun), - phase: mockPhase, - project: getOrCreateProject('2'), - logFilenameIdentifier: '2' - }); - - secondOperation.addDependency(firstOperation); - - const manager: OperationExecutionManager = new OperationExecutionManager( - new Set([firstOperation, secondOperation]), - { - quietMode: false, - debugMode: false, - parallelism: 1, - allowOversubscription: true, - destination: mockWritable - } - ); - - const abortController = new AbortController(); - abortController.abort(); - - const result = await manager.executeAsync(abortController); - expect(result.status).toEqual(OperationStatus.Aborted); - expect(mockRun).not.toHaveBeenCalled(); - expect(result.operationResults.size).toEqual(2); - expect(result.operationResults.get(firstOperation)?.status).toEqual(OperationStatus.Aborted); - expect(result.operationResults.get(secondOperation)?.status).toEqual(OperationStatus.Aborted); - }); - }); - - describe('Blocking', () => { - it('Failed operations block', async () => { - const failingOperation = new Operation({ - runner: new MockOperationRunner('fail', async () => { - return OperationStatus.Failure; - }), - phase: mockPhase, - project: getOrCreateProject('fail'), - logFilenameIdentifier: 'fail' - }); - - const blockedRunFn: jest.Mock = jest.fn(); - - const blockedOperation = new Operation({ - runner: new MockOperationRunner('blocked', blockedRunFn), - phase: mockPhase, - project: getOrCreateProject('blocked'), - logFilenameIdentifier: 'blocked' - }); - - blockedOperation.addDependency(failingOperation); - - const manager: OperationExecutionManager = new OperationExecutionManager( - new Set([failingOperation, blockedOperation]), - { - quietMode: false, - debugMode: false, - parallelism: 1, - allowOversubscription: true, - destination: mockWritable - } - ); - - const abortController = new AbortController(); - const result = await manager.executeAsync(abortController); - expect(result.status).toEqual(OperationStatus.Failure); - expect(blockedRunFn).not.toHaveBeenCalled(); - expect(result.operationResults.size).toEqual(2); - expect(result.operationResults.get(failingOperation)?.status).toEqual(OperationStatus.Failure); - expect(result.operationResults.get(blockedOperation)?.status).toEqual(OperationStatus.Blocked); - }); - }); - - describe('Warning logging', () => { - describe('Fail on warning', () => { - beforeEach(() => { - executionManagerOptions = { - quietMode: false, - debugMode: false, - parallelism: 1, - allowOversubscription: true, - destination: mockWritable - }; - }); - - it('Logs warnings correctly', async () => { - executionManager = createExecutionManager( - executionManagerOptions, - new MockOperationRunner('success with warnings (failure)', async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1\n'); - terminal.writeStdoutLine('Warning: step 1 succeeded with warnings\n'); - return OperationStatus.SuccessWithWarning; - }) - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await executionManager.executeAsync(abortController); - _printOperationStatus(mockTerminal, result); - expect(result.status).toEqual(OperationStatus.SuccessWithWarning); - expect(result.operationResults.size).toEqual(1); - const firstResult: IOperationExecutionResult | undefined = result.operationResults - .values() - .next().value; - expect(firstResult?.status).toEqual(OperationStatus.SuccessWithWarning); - - const allMessages: string = mockWritable.getAllOutput(); - expect(allMessages).toContain('Build step 1'); - expect(allMessages).toContain('step 1 succeeded with warnings'); - expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); - }); - }); - - describe('Success on warning', () => { - beforeEach(() => { - executionManagerOptions = { - quietMode: false, - debugMode: false, - parallelism: 1, - allowOversubscription: true, - destination: mockWritable - }; - }); - - it('Logs warnings correctly', async () => { - executionManager = createExecutionManager( - executionManagerOptions, - new MockOperationRunner( - 'success with warnings (success)', - async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1\n'); - terminal.writeStdoutLine('Warning: step 1 succeeded with warnings\n'); - return OperationStatus.SuccessWithWarning; - }, - /* warningsAreAllowed */ true - ) - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await executionManager.executeAsync(abortController); - _printOperationStatus(mockTerminal, result); - expect(result.status).toEqual(OperationStatus.Success); - expect(result.operationResults.size).toEqual(1); - const firstResult: IOperationExecutionResult | undefined = result.operationResults - .values() - .next().value; - expect(firstResult?.status).toEqual(OperationStatus.SuccessWithWarning); - const allMessages: string = mockWritable.getAllOutput(); - expect(allMessages).toContain('Build step 1'); - expect(allMessages).toContain('Warning: step 1 succeeded with warnings'); - expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); - }); - - it('logs warnings correctly with --timeline option', async () => { - executionManager = createExecutionManager( - executionManagerOptions, - new MockOperationRunner( - 'success with warnings (success)', - async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1\n'); - terminal.writeStdoutLine('Warning: step 1 succeeded with warnings\n'); - return OperationStatus.SuccessWithWarning; - }, - /* warningsAreAllowed */ true - ) - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await executionManager.executeAsync(abortController); - _printTimeline({ terminal: mockTerminal, result, cobuildConfiguration: undefined }); - _printOperationStatus(mockTerminal, result); - const allMessages: string = mockWritable.getAllOutput(); - expect(allMessages).toContain('Build step 1'); - expect(allMessages).toContain('Warning: step 1 succeeded with warnings'); - expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); - }); - }); - }); - - describe('Cobuild logging', () => { - beforeEach(() => { - let mockCobuildTimeInMs: number = 0; - mockGetTimeInMs.mockImplementation(() => { - mockCobuildTimeInMs += 10_000; - return mockCobuildTimeInMs; - }); - }); - function createCobuildExecutionManager( - cobuildExecutionManagerOptions: IOperationExecutionManagerOptions, - operationRunnerFactory: (name: string) => IOperationRunner, - phase: IPhase, - project: RushConfigurationProject - ): OperationExecutionManager { - const operation: Operation = new Operation({ - runner: operationRunnerFactory('operation'), - logFilenameIdentifier: 'operation', - phase, - project - }); - - const operation2: Operation = new Operation({ - runner: operationRunnerFactory('operation2'), - logFilenameIdentifier: 'operation2', - phase, - project - }); - - return new OperationExecutionManager(new Set([operation, operation2]), { - afterExecuteOperationAsync: async (record) => { - if (!record._operationMetadataManager) { - throw new Error('OperationMetadataManager is not defined'); - } - // Mock the readonly state property. - (record._operationMetadataManager as unknown as Record).stateFile = { - state: { - cobuildContextId: '123', - cobuildRunnerId: '456', - nonCachedDurationMs: 15_000 - } - } as unknown as OperationStateFile; - record._operationMetadataManager.wasCobuilt = true; - }, - ...cobuildExecutionManagerOptions - }); - } - it('logs cobuilt operations correctly with --timeline option', async () => { - executionManager = createCobuildExecutionManager( - executionManagerOptions, - (name) => - new MockOperationRunner( - `${name} (success)`, - async () => { - return OperationStatus.Success; - }, - /* warningsAreAllowed */ true - ), - { name: 'my-name' } as unknown as IPhase, - {} as unknown as RushConfigurationProject - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await executionManager.executeAsync(abortController); - _printTimeline({ - terminal: mockTerminal, - result, - cobuildConfiguration: { - cobuildRunnerId: '123', - cobuildContextId: '123' - } as unknown as CobuildConfiguration - }); - _printOperationStatus(mockTerminal, result); - expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); - }); - it('logs warnings correctly with --timeline option', async () => { - executionManager = createCobuildExecutionManager( - executionManagerOptions, - (name) => - new MockOperationRunner(`${name} (success with warnings)`, async (terminal: CollatedTerminal) => { - terminal.writeStdoutLine('Build step 1\n'); - terminal.writeStdoutLine('Warning: step 1 succeeded with warnings\n'); - return OperationStatus.SuccessWithWarning; - }), - { name: 'my-name' } as unknown as IPhase, - {} as unknown as RushConfigurationProject - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await executionManager.executeAsync(abortController); - _printTimeline({ - terminal: mockTerminal, - result, - cobuildConfiguration: { - cobuildRunnerId: '123', - cobuildContextId: '123' - } as unknown as CobuildConfiguration - }); - _printOperationStatus(mockTerminal, result); - const allMessages: string = mockWritable.getAllOutput(); - expect(allMessages).toContain('Build step 1'); - expect(allMessages).toContain('Warning: step 1 succeeded with warnings'); - expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); - }); - }); - - describe('Weighted concurrency', () => { - function createWeightedOperation( - name: string, - weight: number, - counters: { concurrentCount: number; peakConcurrency: number } - ): Operation { - const operation: Operation = new Operation({ - runner: new MockOperationRunner(name, async (terminal: CollatedTerminal) => { - counters.concurrentCount++; - if (counters.concurrentCount > counters.peakConcurrency) { - counters.peakConcurrency = counters.concurrentCount; - } - await Async.sleepAsync(0); - if (counters.concurrentCount > counters.peakConcurrency) { - counters.peakConcurrency = counters.concurrentCount; - } - counters.concurrentCount--; - return OperationStatus.Success; - }), - phase: mockPhase, - project: getOrCreateProject(name), - logFilenameIdentifier: name - }); - operation.weight = weight; - return operation; - } - - it('does not cap the unit budget by the number of operations (issue #5607 regression)', async () => { - // Regression test for https://github.com/microsoft/rushstack/issues/5607 - // With weighted scheduling, concurrency is a unit budget. The old code passed - // Math.min(totalOperations, parallelism), which shrinks the budget when - // totalOperations < parallelism, causing serialization for weight > 1. - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const opA: Operation = createWeightedOperation('A', 4, counters); - const opB: Operation = createWeightedOperation('B', 4, counters); - const opC: Operation = createWeightedOperation('C', 4, counters); - const opD: Operation = createWeightedOperation('D', 4, counters); - - const manager: OperationExecutionManager = new OperationExecutionManager( - new Set([opA, opB, opC, opD]), - { - quietMode: true, - debugMode: false, - parallelism: 10, - allowOversubscription: false, - destination: mockWritable - } - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await manager.executeAsync(abortController); - - expect(result.status).toEqual(OperationStatus.Success); - expect(counters.peakConcurrency).toEqual(2); - }); - - it('clamps weight to budget and completes without deadlock when weight exceeds budget', async () => { - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const opA: Operation = createWeightedOperation('heavy-A', 10, counters); - const opB: Operation = createWeightedOperation('heavy-B', 10, counters); - - const manager: OperationExecutionManager = new OperationExecutionManager( - new Set([opA, opB]), - { - quietMode: true, - debugMode: false, - parallelism: 4, - allowOversubscription: false, - destination: mockWritable - } - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await manager.executeAsync(abortController); - - expect(result.status).toEqual(OperationStatus.Success); - expect(result.operationResults.get(opA)?.status).toEqual(OperationStatus.Success); - expect(result.operationResults.get(opB)?.status).toEqual(OperationStatus.Success); - expect(counters.peakConcurrency).toEqual(1); - }); - - it('allows oversubscription when allowOversubscription is true', async () => { - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const opA: Operation = createWeightedOperation('over-A', 7, counters); - const opB: Operation = createWeightedOperation('over-B', 7, counters); - - const manager: OperationExecutionManager = new OperationExecutionManager( - new Set([opA, opB]), - { - quietMode: true, - debugMode: false, - parallelism: 10, - allowOversubscription: true, - destination: mockWritable - } - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await manager.executeAsync(abortController); - - expect(result.status).toEqual(OperationStatus.Success); - expect(counters.peakConcurrency).toEqual(2); - }); - - it('does not oversubscribe when allowOversubscription is false', async () => { - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const opA: Operation = createWeightedOperation('strict-A', 7, counters); - const opB: Operation = createWeightedOperation('strict-B', 7, counters); - - const manager: OperationExecutionManager = new OperationExecutionManager( - new Set([opA, opB]), - { - quietMode: true, - debugMode: false, - parallelism: 10, - allowOversubscription: false, - destination: mockWritable - } - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await manager.executeAsync(abortController); - - expect(result.status).toEqual(OperationStatus.Success); - expect(counters.peakConcurrency).toEqual(1); - }); - - it('zero-weight operations do not consume budget', async () => { - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const heavyOp: Operation = createWeightedOperation('heavy', 9, counters); - const zeroA: Operation = createWeightedOperation('zero-A', 0, counters); - const zeroB: Operation = createWeightedOperation('zero-B', 0, counters); - const zeroC: Operation = createWeightedOperation('zero-C', 0, counters); - - const manager: OperationExecutionManager = new OperationExecutionManager( - new Set([heavyOp, zeroA, zeroB, zeroC]), - { - quietMode: true, - debugMode: false, - parallelism: 10, - allowOversubscription: false, - destination: mockWritable - } - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await manager.executeAsync(abortController); - - expect(result.status).toEqual(OperationStatus.Success); - expect(counters.peakConcurrency).toBeGreaterThanOrEqual(2); - }); - - it('mixed weights respect the unit budget correctly', async () => { - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const opA: Operation = createWeightedOperation('mix-A', 5, counters); - const opB: Operation = createWeightedOperation('mix-B', 5, counters); - const opC: Operation = createWeightedOperation('mix-C', 3, counters); - const opD: Operation = createWeightedOperation('mix-D', 3, counters); - - const manager: OperationExecutionManager = new OperationExecutionManager( - new Set([opA, opB, opC, opD]), - { - quietMode: true, - debugMode: false, - parallelism: 10, - allowOversubscription: false, - destination: mockWritable - } - ); - - const abortController = new AbortController(); - const result: IExecutionResult = await manager.executeAsync(abortController); - - expect(result.status).toEqual(OperationStatus.Success); - for (const [, opResult] of result.operationResults) { - expect(opResult.status).toEqual(OperationStatus.Success); - } - expect(counters.peakConcurrency).toBeGreaterThanOrEqual(2); - expect(counters.peakConcurrency).toBeLessThanOrEqual(3); - }); - - it('weight=1 operations behave identically to unweighted scheduling', async () => { - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const ops: Operation[] = []; - for (let i = 0; i < 5; i++) { - ops.push(createWeightedOperation(`unit-${i}`, 1, counters)); - } - - const manager: OperationExecutionManager = new OperationExecutionManager(new Set(ops), { - quietMode: true, - debugMode: false, - parallelism: 3, - allowOversubscription: false, - destination: mockWritable - }); - - const abortController = new AbortController(); - const result: IExecutionResult = await manager.executeAsync(abortController); - - expect(result.status).toEqual(OperationStatus.Success); - expect(counters.peakConcurrency).toEqual(3); - }); - - it('displays the capped process count when parallelism exceeds operation count', async () => { - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const ops: Operation[] = []; - for (let i = 0; i < 4; i++) { - ops.push(createWeightedOperation(`log-${i}`, 4, counters)); - } - - const manager: OperationExecutionManager = new OperationExecutionManager(new Set(ops), { - quietMode: false, - debugMode: false, - parallelism: 10, - allowOversubscription: false, - destination: mockWritable - }); - - const abortController = new AbortController(); - await manager.executeAsync(abortController); - - const allOutput: string = mockWritable.getAllOutput(); - expect(allOutput).toContain('Executing a maximum of 4 simultaneous processes...'); - expect(allOutput).not.toContain('Executing a maximum of 10 simultaneous processes...'); - }); - - it('displays parallelism when it is less than operation count', async () => { - const counters = { concurrentCount: 0, peakConcurrency: 0 }; - - const ops: Operation[] = []; - for (let i = 0; i < 10; i++) { - ops.push(createWeightedOperation(`many-${i}`, 1, counters)); - } - - const manager: OperationExecutionManager = new OperationExecutionManager(new Set(ops), { - quietMode: false, - debugMode: false, - parallelism: 3, - allowOversubscription: false, - destination: mockWritable - }); - - const abortController = new AbortController(); - await manager.executeAsync(abortController); - - const allOutput: string = mockWritable.getAllOutput(); - expect(allOutput).toContain('Executing a maximum of 3 simultaneous processes...'); - }); - }); -}); diff --git a/libraries/rush-lib/src/logic/operations/test/OperationExecutionRecord.test.ts b/libraries/rush-lib/src/logic/operations/test/OperationExecutionRecord.test.ts new file mode 100644 index 00000000000..6919cc704bf --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/test/OperationExecutionRecord.test.ts @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import type { IPhase } from '../../../api/CommandLineConfiguration'; +import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; +import type { IOperationSettings } from '../../../api/RushProjectConfiguration'; +import { Operation } from '../Operation'; +import { type IOperationExecutionRecordContext, OperationExecutionRecord } from '../OperationExecutionRecord'; +import { MockOperationRunner } from './MockOperationRunner'; + +const MOCK_PHASE: IPhase = { + name: '_phase:test', + allowWarningsOnSuccess: false, + associatedParameters: new Set(), + dependencies: { + self: new Set(), + upstream: new Set() + }, + isSynthetic: false, + logFilenameIdentifier: '_phase_test', + missingScriptBehavior: 'silent' +}; + +function createProject(packageName: string): RushConfigurationProject { + return { + packageName + } as RushConfigurationProject; +} + +function createOperation(options: { + project: RushConfigurationProject; + settings?: IOperationSettings; + isNoOp?: boolean; +}): Operation { + const { project, settings, isNoOp } = options; + return new Operation({ + phase: MOCK_PHASE, + project, + settings, + runner: new MockOperationRunner(`${project.packageName} (${MOCK_PHASE.name})`, undefined, false, isNoOp), + logFilenameIdentifier: `${project.packageName}_phase_test` + }); +} + +function createRecord(operation: Operation, maxParallelism: number = 8): OperationExecutionRecord { + return new OperationExecutionRecord(operation, { + maxParallelism + } as unknown as IOperationExecutionRecordContext); +} + +describe(OperationExecutionRecord.name, () => { + describe('weight', () => { + it('snapshots numeric operation weight for a normal (non-no-op) operation', () => { + const project: RushConfigurationProject = createProject('project-normal'); + const operation: Operation = createOperation({ + project, + settings: { + operationName: MOCK_PHASE.name, + weight: 3 + } + }); + + const record: OperationExecutionRecord = createRecord(operation); + expect(record.weight).toBe(3); + }); + + it('coerces percentage weight to integer slots using maxParallelism', () => { + // 25% of 8 slots = floor(0.25 * 8) = 2 + const project: RushConfigurationProject = createProject('project-percent'); + const operation: Operation = createOperation({ + project, + settings: { + operationName: MOCK_PHASE.name, + weight: '25%' + } as IOperationSettings + }); + + const record: OperationExecutionRecord = createRecord(operation, 8); + expect(record.weight).toBe(2); + }); + + it('coerces weight to 0 for no-op operations regardless of operation weight', () => { + const project: RushConfigurationProject = createProject('project-noop'); + const operation: Operation = createOperation({ + project, + settings: { + operationName: MOCK_PHASE.name, + weight: 5 + }, + isNoOp: true + }); + + const record: OperationExecutionRecord = createRecord(operation); + expect(record.weight).toBe(0); + }); + + it('snapshots default weight (1) for a normal operation with no weight setting', () => { + const project: RushConfigurationProject = createProject('project-default'); + const operation: Operation = createOperation({ project }); + + const record: OperationExecutionRecord = createRecord(operation); + expect(record.weight).toBe(1); + }); + + it('coerces weight to 0 for no-op operations even with default weight', () => { + const project: RushConfigurationProject = createProject('project-noop-default'); + const operation: Operation = createOperation({ project, isNoOp: true }); + + const record: OperationExecutionRecord = createRecord(operation); + expect(record.weight).toBe(0); + }); + + it('uses the graph maxParallelism (not OS core count) when coercing percentage weights', () => { + // 50% of 4 slots = floor(0.5 * 4) = 2, not floor(0.5 * ) + const project: RushConfigurationProject = createProject('project-graph-max'); + const operation: Operation = createOperation({ + project, + settings: { + operationName: MOCK_PHASE.name, + weight: '50%' + } as IOperationSettings + }); + + const record: OperationExecutionRecord = createRecord(operation, 4); + expect(record.weight).toBe(2); + }); + }); +}); diff --git a/libraries/rush-lib/src/logic/operations/test/OperationGraph.test.ts b/libraries/rush-lib/src/logic/operations/test/OperationGraph.test.ts new file mode 100644 index 00000000000..2110b0f52a3 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/test/OperationGraph.test.ts @@ -0,0 +1,1386 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +// The TaskExecutionManager prints "x.xx seconds" in TestRunner.test.ts.snap; ensure that the Stopwatch timing is deterministic +jest.mock('@rushstack/terminal', () => { + const originalModule = jest.requireActual('@rushstack/terminal'); + return { + ...originalModule, + ConsoleTerminalProvider: { + ...originalModule.ConsoleTerminalProvider, + supportsColor: true + } + }; +}); + +jest.mock('../../../utilities/Utilities'); +jest.mock('../OperationStateFile'); +// Mock project log file creation to avoid filesystem writes; return a simple writable collecting chunks. +jest.mock('../ProjectLogWritable', () => { + const actual = jest.requireActual('../ProjectLogWritable'); + const terminalModule = jest.requireActual('@rushstack/terminal'); + const { TerminalWritable } = terminalModule; + class MockTerminalWritable extends TerminalWritable { + public readonly chunks: string[] = []; + protected onWriteChunk(chunk: { text: string }): void { + this.chunks.push(chunk.text); + } + protected onClose(): void { + /* noop */ + } + } + return { + ...actual, + initializeProjectLogFilesAsync: jest.fn(async () => new MockTerminalWritable()) + }; +}); + +import { type ITerminal, Terminal } from '@rushstack/terminal'; +import { CollatedTerminal } from '@rushstack/stream-collator'; +import { MockWritable, PrintUtilities } from '@rushstack/terminal'; +import { Async } from '@rushstack/node-core-library'; + +import type { IPhase } from '../../../api/CommandLineConfiguration'; +import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; +import { OperationGraph, type IOperationGraphOptions } from '../OperationGraph'; +import { _printOperationStatus } from '../OperationResultSummarizerPlugin'; +import { _printTimeline } from '../ConsoleTimelinePlugin'; +import { OperationStatus } from '../OperationStatus'; +import { Operation } from '../Operation'; +import { Utilities } from '../../../utilities/Utilities'; +import type { IOperationRunner, IOperationRunnerContext } from '../IOperationRunner'; +import { MockOperationRunner } from './MockOperationRunner'; +import type { IExecutionResult, IOperationExecutionResult } from '../IOperationExecutionResult'; +import { CollatedTerminalProvider } from '../../../utilities/CollatedTerminalProvider'; +import type { CobuildConfiguration } from '../../../api/CobuildConfiguration'; +import type { OperationStateFile } from '../OperationStateFile'; +import type { IOperationGraphIterationOptions } from '../IOperationGraph'; +import type { IOperationGraph } from '../IOperationGraph'; + +const mockGetTimeInMs: jest.Mock = jest.fn(); +Utilities.getTimeInMs = mockGetTimeInMs; + +let mockTimeInMs: number = 0; +mockGetTimeInMs.mockImplementation(() => { + mockTimeInMs += 100; + return mockTimeInMs; +}); + +const mockWritable: MockWritable = new MockWritable(); +const mockTerminal: Terminal = new Terminal(new CollatedTerminalProvider(new CollatedTerminal(mockWritable))); + +const mockPhase: IPhase = { + name: 'phase', + allowWarningsOnSuccess: false, + associatedParameters: new Set(), + dependencies: { + self: new Set(), + upstream: new Set() + }, + isSynthetic: false, + logFilenameIdentifier: 'phase', + missingScriptBehavior: 'silent' +}; +const projectsByName: Map = new Map(); +function getOrCreateProject(name: string): RushConfigurationProject { + let project: RushConfigurationProject | undefined = projectsByName.get(name); + if (!project) { + project = { + packageName: name + } as unknown as RushConfigurationProject; + projectsByName.set(name, project); + } + return project; +} + +function createGraph( + graphOptions: IOperationGraphOptions, + operationRunner: IOperationRunner +): OperationGraph { + const operation: Operation = new Operation({ + runner: operationRunner, + logFilenameIdentifier: 'operation', + phase: mockPhase, + project: getOrCreateProject('project') + }); + + return new OperationGraph(new Set([operation]), graphOptions); +} + +describe('OperationGraph', () => { + let graphOptions: IOperationGraphOptions; + let graphIterationOptions: IOperationGraphIterationOptions; + + beforeEach(() => { + jest.spyOn(PrintUtilities, 'getConsoleWidth').mockReturnValue(90); + mockWritable.reset(); + }); + + describe('Error logging', () => { + beforeEach(() => { + graphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + graphIterationOptions = {}; + }); + + it('printedStderrAfterError', async () => { + const graph: OperationGraph = createGraph( + graphOptions, + new MockOperationRunner('stdout+stderr', async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Build step 1\n'); + terminal.writeStderrLine('Error: step 1 failed\n'); + return OperationStatus.Failure; + }) + ); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + _printOperationStatus(mockTerminal, result); + expect(result.status).toEqual(OperationStatus.Failure); + expect(graph.status).toEqual(OperationStatus.Failure); + expect(result.operationResults.size).toEqual(1); + const firstResult: IOperationExecutionResult | undefined = result.operationResults + .values() + .next().value; + expect(firstResult?.status).toEqual(OperationStatus.Failure); + + const allMessages: string = mockWritable.getAllOutput(); + expect(allMessages).toContain('Error: step 1 failed'); + expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); + }); + + it('printedStdoutAfterErrorWithEmptyStderr', async () => { + const graph: OperationGraph = createGraph( + graphOptions, + new MockOperationRunner('stdout only', async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Build step 1\n'); + terminal.writeStdoutLine('Error: step 1 failed\n'); + return OperationStatus.Failure; + }) + ); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + _printOperationStatus(mockTerminal, result); + expect(result.status).toEqual(OperationStatus.Failure); + expect(result.operationResults.size).toEqual(1); + const firstResult: IOperationExecutionResult | undefined = result.operationResults + .values() + .next().value; + expect(firstResult?.status).toEqual(OperationStatus.Failure); + + const allOutput: string = mockWritable.getAllOutput(); + expect(allOutput).toMatch(/Build step 1/); + expect(allOutput).toMatch(/Error: step 1 failed/); + expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); + }); + }); + + describe('Aborting', () => { + it('Aborted operations abort', async () => { + const mockRun: jest.Mock = jest.fn(); + + const firstOperation = new Operation({ + runner: new MockOperationRunner('1', mockRun), + phase: mockPhase, + project: getOrCreateProject('1'), + logFilenameIdentifier: '1' + }); + + const secondOperation = new Operation({ + runner: new MockOperationRunner('2', mockRun), + phase: mockPhase, + project: getOrCreateProject('2'), + logFilenameIdentifier: '2' + }); + + secondOperation.addDependency(firstOperation); + + const graph: OperationGraph = new OperationGraph(new Set([firstOperation, secondOperation]), { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }); + + graph.hooks.beforeExecuteIterationAsync.tapPromise( + 'test', + (): Promise => graph.abortCurrentIterationAsync() + ); + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + expect(result.status).toEqual(OperationStatus.Aborted); + expect(graph.status).toEqual(OperationStatus.Aborted); + expect(mockRun).not.toHaveBeenCalled(); + expect(result.operationResults.size).toEqual(2); + expect(result.operationResults.get(firstOperation)?.status).toEqual(OperationStatus.Aborted); + expect(result.operationResults.get(secondOperation)?.status).toEqual(OperationStatus.Aborted); + }); + }); + + describe('Blocking', () => { + it('Failed operations block', async () => { + const failingOperation = new Operation({ + runner: new MockOperationRunner('fail', async () => { + return OperationStatus.Failure; + }), + phase: mockPhase, + project: getOrCreateProject('fail'), + logFilenameIdentifier: 'fail' + }); + + const blockedRunFn: jest.Mock = jest.fn(); + + const blockedOperation = new Operation({ + runner: new MockOperationRunner('blocked', blockedRunFn), + phase: mockPhase, + project: getOrCreateProject('blocked'), + logFilenameIdentifier: 'blocked' + }); + + blockedOperation.addDependency(failingOperation); + + const graph: OperationGraph = new OperationGraph(new Set([failingOperation, blockedOperation]), { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }); + + const result = await graph.executeAsync({}); + expect(result.status).toEqual(OperationStatus.Failure); + expect(blockedRunFn).not.toHaveBeenCalled(); + expect(result.operationResults.size).toEqual(2); + expect(result.operationResults.get(failingOperation)?.status).toEqual(OperationStatus.Failure); + expect(result.operationResults.get(blockedOperation)?.status).toEqual(OperationStatus.Blocked); + }); + }); + + describe('Concurrency', () => { + it('runs independent operations concurrently when parallelism allows', async () => { + let concurrency: number = 0; + let maxConcurrency: number = 0; + + const trackingRun = async (): Promise => { + ++concurrency; + await Async.sleepAsync(0); + if (concurrency > maxConcurrency) { + maxConcurrency = concurrency; + } + --concurrency; + return OperationStatus.Success; + }; + + const alpha = new Operation({ + runner: new MockOperationRunner('alpha', trackingRun), + phase: mockPhase, + project: getOrCreateProject('alpha'), + logFilenameIdentifier: 'alpha' + }); + const beta = new Operation({ + runner: new MockOperationRunner('beta', trackingRun), + phase: mockPhase, + project: getOrCreateProject('beta'), + logFilenameIdentifier: 'beta' + }); + + const graph: OperationGraph = new OperationGraph(new Set([alpha, beta]), { + quietMode: false, + debugMode: false, + parallelism: 2, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }); + + const result: IExecutionResult = await graph.executeAsync({}); + expect(result.status).toEqual(OperationStatus.Success); + expect(maxConcurrency).toBe(2); + }); + + it('serializes independent operations when parallelism is 1', async () => { + let concurrency: number = 0; + let maxConcurrency: number = 0; + + const trackingRun = async (): Promise => { + ++concurrency; + await Async.sleepAsync(0); + if (concurrency > maxConcurrency) { + maxConcurrency = concurrency; + } + --concurrency; + return OperationStatus.Success; + }; + + const alpha = new Operation({ + runner: new MockOperationRunner('alpha-seq', trackingRun), + phase: mockPhase, + project: getOrCreateProject('alpha-seq'), + logFilenameIdentifier: 'alpha-seq' + }); + const beta = new Operation({ + runner: new MockOperationRunner('beta-seq', trackingRun), + phase: mockPhase, + project: getOrCreateProject('beta-seq'), + logFilenameIdentifier: 'beta-seq' + }); + + const graph: OperationGraph = new OperationGraph(new Set([alpha, beta]), { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }); + + const result: IExecutionResult = await graph.executeAsync({}); + expect(result.status).toEqual(OperationStatus.Success); + expect(maxConcurrency).toBe(1); + }); + }); + + describe('onExecutionStatesUpdated hook', () => { + beforeEach(() => { + graphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + graphIterationOptions = {}; + }); + + class LogFileCreatingRunner extends MockOperationRunner { + public constructor() { + super('logfile-op'); + } + public override async executeAsync(context: IOperationRunnerContext): Promise { + await context.runWithTerminalAsync( + async (terminal: ITerminal) => { + terminal.writeLine('Hello world'); + return Promise.resolve(); + }, + { createLogFile: true, logFileSuffix: '' } + ); + return OperationStatus.Success; + } + } + + it('fires state updates for status transitions (captures snapshot statuses)', async () => { + const runner: IOperationRunner = new MockOperationRunner('state-change-op'); + const graph: OperationGraph = createGraph(graphOptions, runner); + + const stateUpdates: OperationStatus[][] = []; + graph.hooks.onExecutionStatesUpdated.tap('test', (records) => { + // Capture immutable array of status values at callback time + stateUpdates.push(Array.from(records, (r) => r.status)); + }); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + expect(result.status).toBe(OperationStatus.Success); + // Expect at least two batches now that we introduced a delay + expect(stateUpdates.length).toBeGreaterThanOrEqual(2); + const flattenedStatuses: OperationStatus[] = stateUpdates.flat(); + // Should observe an Executing intermediate status in snapshots (not just final Success) + expect(flattenedStatuses).toContain(OperationStatus.Executing); + expect(flattenedStatuses).toContain(OperationStatus.Success); + }); + + it('fires state update when logFilePaths are assigned (createLogFile=true) regardless of final status', async () => { + const runner: IOperationRunner = new LogFileCreatingRunner(); + const graph: OperationGraph = createGraph(graphOptions, runner); + + const operationStateUpdates: ReadonlySet[] = []; + graph.hooks.onExecutionStatesUpdated.tap('test', (records) => { + operationStateUpdates.push(new Set(records)); + }); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + // Status may be Success or Failure if logging pipeline errors; we only care that hook fired with logFilePaths + expect(result.status === OperationStatus.Success || result.status === OperationStatus.Failure).toBe( + true + ); + // Find a batch where logFilePaths is defined + const anyWithLogFile: boolean = operationStateUpdates.some((recordSet) => + Array.from(recordSet).some((r) => Boolean((r as { logFilePaths?: unknown }).logFilePaths)) + ); + expect(anyWithLogFile).toBe(true); + }); + }); + + describe('Warning logging', () => { + describe('Fail on warning', () => { + beforeEach(() => { + graphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + }); + + it('Logs warnings correctly', async () => { + const graph: OperationGraph = createGraph( + graphOptions, + new MockOperationRunner('success with warnings (failure)', async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Build step 1\n'); + terminal.writeStdoutLine('Warning: step 1 succeeded with warnings\n'); + return OperationStatus.SuccessWithWarning; + }) + ); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + _printOperationStatus(mockTerminal, result); + expect(result.status).toEqual(OperationStatus.SuccessWithWarning); + expect(graph.status).toEqual(OperationStatus.SuccessWithWarning); + expect(result.operationResults.size).toEqual(1); + const firstResult: IOperationExecutionResult | undefined = result.operationResults + .values() + .next().value; + expect(firstResult?.status).toEqual(OperationStatus.SuccessWithWarning); + + const allMessages: string = mockWritable.getAllOutput(); + expect(allMessages).toContain('Build step 1'); + expect(allMessages).toContain('step 1 succeeded with warnings'); + expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); + }); + }); + + describe('Success on warning', () => { + beforeEach(() => { + graphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + }); + + it('Logs warnings correctly', async () => { + const graph: OperationGraph = createGraph( + graphOptions, + new MockOperationRunner( + 'success with warnings (success)', + async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Build step 1\n'); + terminal.writeStdoutLine('Warning: step 1 succeeded with warnings\n'); + return OperationStatus.SuccessWithWarning; + }, + /* warningsAreAllowed */ true + ) + ); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + _printOperationStatus(mockTerminal, result); + expect(result.status).toEqual(OperationStatus.Success); + expect(result.operationResults.size).toEqual(1); + const firstResult: IOperationExecutionResult | undefined = result.operationResults + .values() + .next().value; + expect(firstResult?.status).toEqual(OperationStatus.SuccessWithWarning); + const allMessages: string = mockWritable.getAllOutput(); + expect(allMessages).toContain('Build step 1'); + expect(allMessages).toContain('Warning: step 1 succeeded with warnings'); + expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); + }); + + it('logs warnings correctly with --timeline option', async () => { + const graph: OperationGraph = createGraph( + graphOptions, + new MockOperationRunner( + 'success with warnings (success)', + async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Build step 1\n'); + terminal.writeStdoutLine('Warning: step 1 succeeded with warnings\n'); + return OperationStatus.SuccessWithWarning; + }, + /* warningsAreAllowed */ true + ) + ); + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + _printTimeline({ terminal: mockTerminal, result, cobuildConfiguration: undefined }); + _printOperationStatus(mockTerminal, result); + const allMessages: string = mockWritable.getAllOutput(); + expect(allMessages).toContain('Build step 1'); + expect(allMessages).toContain('Warning: step 1 succeeded with warnings'); + expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); + }); + }); + }); + + describe('Cobuild logging', () => { + beforeEach(() => { + let mockCobuildTimeInMs: number = 0; + mockGetTimeInMs.mockImplementation(() => { + mockCobuildTimeInMs += 10_000; + return mockCobuildTimeInMs; + }); + }); + + function createCobuildGraph( + cobuildExecutionManagerOptions: IOperationGraphOptions, + operationRunnerFactory: (name: string) => IOperationRunner, + phase: IPhase, + project: RushConfigurationProject + ): OperationGraph { + const operation: Operation = new Operation({ + runner: operationRunnerFactory('operation'), + logFilenameIdentifier: 'operation', + phase, + project + }); + + const operation2: Operation = new Operation({ + runner: operationRunnerFactory('operation2'), + logFilenameIdentifier: 'operation2', + phase, + project + }); + + const graph: OperationGraph = new OperationGraph( + new Set([operation, operation2]), + cobuildExecutionManagerOptions + ); + + graph.hooks.afterExecuteOperationAsync.tapPromise('TestPlugin', async (record) => { + if (!record._operationMetadataManager) { + throw new Error('OperationMetadataManager is not defined'); + } + // Mock the readonly state property. + (record._operationMetadataManager as unknown as Record).stateFile = { + state: { + cobuildContextId: '123', + cobuildRunnerId: '456', + nonCachedDurationMs: 15_000 + } + } as unknown as OperationStateFile; + record._operationMetadataManager.wasCobuilt = true; + }); + + return graph; + } + it('logs cobuilt operations correctly with --timeline option', async () => { + const graph: OperationGraph = createCobuildGraph( + graphOptions, + (name) => + new MockOperationRunner( + `${name} (success)`, + async () => { + return OperationStatus.Success; + }, + /* warningsAreAllowed */ true + ), + { name: 'my-name' } as unknown as IPhase, + {} as unknown as RushConfigurationProject + ); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + _printTimeline({ + terminal: mockTerminal, + result, + cobuildConfiguration: { + cobuildRunnerId: '123', + cobuildContextId: '123' + } as unknown as CobuildConfiguration + }); + _printOperationStatus(mockTerminal, result); + expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); + }); + it('logs warnings correctly with --timeline option', async () => { + const graph: OperationGraph = createCobuildGraph( + graphOptions, + (name) => + new MockOperationRunner(`${name} (success with warnings)`, async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Build step 1\n'); + terminal.writeStdoutLine('Warning: step 1 succeeded with warnings\n'); + return OperationStatus.SuccessWithWarning; + }), + { name: 'my-name' } as unknown as IPhase, + {} as unknown as RushConfigurationProject + ); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + _printTimeline({ + terminal: mockTerminal, + result, + cobuildConfiguration: { + cobuildRunnerId: '123', + cobuildContextId: '123' + } as unknown as CobuildConfiguration + }); + _printOperationStatus(mockTerminal, result); + const allMessages: string = mockWritable.getAllOutput(); + expect(allMessages).toContain('Build step 1'); + expect(allMessages).toContain('Warning: step 1 succeeded with warnings'); + expect(mockWritable.getFormattedChunks()).toMatchSnapshot(); + }); + }); + + describe('Manual iteration mode', () => { + it('queues an iteration in manual mode and does not auto-execute until executeScheduledIterationAsync is called', async () => { + jest.useFakeTimers({ legacyFakeTimers: true }); + try { + const options: IOperationGraphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController(), + pauseNextIteration: true + }; + + const runFn: jest.Mock = jest.fn(async () => OperationStatus.Success); + const op: Operation = new Operation({ + runner: new MockOperationRunner('manual-op', runFn), + phase: mockPhase, + project: getOrCreateProject('manual-project'), + logFilenameIdentifier: 'manual-op' + }); + + const graph: OperationGraph = new OperationGraph(new Set([op]), options); + + const passQueuedCalls: ReadonlyMap[] = []; + graph.hooks.onIterationScheduled.tap('test', (records) => passQueuedCalls.push(records)); + + const idleCalls: number[] = []; + graph.hooks.onIdle.tap('test', () => idleCalls.push(1)); + + const queued: boolean = await graph.scheduleIterationAsync({}); + expect(queued).toBe(true); + expect(passQueuedCalls.length).toBe(1); + // Iteration should be scheduled but not yet started. + expect(graph.hasScheduledIteration).toBe(true); + expect(runFn).not.toHaveBeenCalled(); + + // Flush the idle timeout. Since pauseNextIteration is true, execution should NOT start automatically. + jest.runAllTimers(); + expect(idleCalls.length).toBe(1); + expect(runFn).not.toHaveBeenCalled(); + + // Now manually execute the scheduled iteration + const executed: boolean = await graph.executeScheduledIterationAsync(); + expect(executed).toBe(true); + expect(runFn).toHaveBeenCalledTimes(1); + expect(graph.hasScheduledIteration).toBe(false); + // After execution status should be Success + expect(graph.status).toBe(OperationStatus.Success); + } finally { + jest.useRealTimers(); + } + }); + + it('does not queue an iteration if all operations are disabled (no enabled operations)', async () => { + const options: IOperationGraphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController(), + pauseNextIteration: true + }; + + const runFn: jest.Mock = jest.fn(async () => OperationStatus.Success); + const disabledOp: Operation = new Operation({ + runner: new MockOperationRunner('disabled-op', runFn), + phase: mockPhase, + project: getOrCreateProject('disabled-project'), + logFilenameIdentifier: 'disabled-op', + enabled: false + }); + + const graph: OperationGraph = new OperationGraph(new Set([disabledOp]), options); + + const passQueuedCalls: ReadonlyMap[] = []; + graph.hooks.onIterationScheduled.tap('test', (records) => passQueuedCalls.push(records)); + + const queued: boolean = await graph.scheduleIterationAsync({}); + expect(queued).toBe(false); // Nothing to do + expect(passQueuedCalls.length).toBe(0); // Hook not fired + expect(graph.hasScheduledIteration).toBe(false); + expect(runFn).not.toHaveBeenCalled(); + // Status remains Ready (no operations executed) + expect(graph.status).toBe(OperationStatus.Ready); + }); + }); + + describe('Terminal destination APIs', () => { + beforeEach(() => { + graphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + graphIterationOptions = {}; + }); + + it('addTerminalDestination causes new destination to receive output', async () => { + const extraDest = new MockWritable(); + + const graph: OperationGraph = createGraph( + graphOptions, + new MockOperationRunner('to-extra', async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Message for extra destination'); + return OperationStatus.Success; + }) + ); + + // Add destination before executing + graph.addTerminalDestination(extraDest); + + const result: IExecutionResult = await graph.executeAsync(graphIterationOptions); + expect(result.status).toBe(OperationStatus.Success); + + const allOutput: string = extraDest.getAllOutput(); + expect(allOutput).toContain('Message for extra destination'); + }); + + it('removeTerminalDestination closes destination by default and stops further output', async () => { + const extraDest = new MockWritable(); + + const graph: OperationGraph = createGraph( + graphOptions, + new MockOperationRunner('to-extra', async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Iteration message'); + return OperationStatus.Success; + }) + ); + + graph.addTerminalDestination(extraDest); + + // First run: destination should receive output + const first = await graph.executeAsync(graphIterationOptions); + expect(first.status).toBe(OperationStatus.Success); + expect(extraDest.getAllOutput()).toContain('Iteration message'); + + // Now remove destination (default close = true) and ensure it was removed/closed + const removed = graph.removeTerminalDestination(extraDest); + expect(removed).toBe(true); + // TerminalWritable exposes isOpen + expect(extraDest.isOpen).toBe(false); + + // Second run: should not write to closed destination + const beforeSecond = extraDest.getAllOutput(); + const second = await graph.executeAsync(graphIterationOptions); + expect(second.status).toBe(OperationStatus.Success); + const afterSecond = extraDest.getAllOutput(); + expect(afterSecond).toBe(beforeSecond); + }); + + it('removeTerminalDestination with close=false does not close destination but still stops further output', async () => { + const extraDest = new MockWritable(); + + const graph: OperationGraph = createGraph( + graphOptions, + new MockOperationRunner('to-extra', async (terminal: CollatedTerminal) => { + terminal.writeStdoutLine('Iteration message 2'); + return OperationStatus.Success; + }) + ); + + graph.addTerminalDestination(extraDest); + + // First run: destination should receive output + const first = await graph.executeAsync(graphIterationOptions); + expect(first.status).toBe(OperationStatus.Success); + expect(extraDest.getAllOutput()).toContain('Iteration message 2'); + + // Remove without closing + const removed = graph.removeTerminalDestination(extraDest, false); + expect(removed).toBe(true); + // Destination should remain open + expect(extraDest.isOpen).toBe(true); + + // Second run: destination should not receive additional output + const beforeSecond = extraDest.getAllOutput(); + const second = await graph.executeAsync(graphIterationOptions); + expect(second.status).toBe(OperationStatus.Success); + const afterSecond = extraDest.getAllOutput(); + expect(afterSecond).toBe(beforeSecond); + }); + + it('removeTerminalDestination returns false when destination not found', () => { + const unknown = new MockWritable(); + const graph = createGraph(graphOptions, new MockOperationRunner('noop')); + const removed = graph.removeTerminalDestination(unknown); + expect(removed).toBe(false); + }); + }); +}); + +describe('invalidateOperations', () => { + it('invalidates a specific operation and updates graph status', async () => { + const graphOptions: IOperationGraphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + + const runner: IOperationRunner = new MockOperationRunner('invalidate-success', async () => { + return OperationStatus.Success; + }); + + const graph: OperationGraph = createGraph(graphOptions, runner); + + const invalidateCalls: Array<{ ops: Iterable; reason: string | undefined }> = []; + graph.hooks.onInvalidateOperations.tap('test', (ops: Iterable, reason: string | undefined) => { + invalidateCalls.push({ ops, reason }); + }); + + const result: IExecutionResult = await graph.executeAsync({}); + expect(result.status).toBe(OperationStatus.Success); + const record: IOperationExecutionResult | undefined = result.operationResults.values().next().value; + expect(record?.status).toBe(OperationStatus.Success); + + const operation: Operation = Array.from(graph.operations)[0]; + graph.invalidateOperations([operation], 'unit-test'); + + const postRecord: IOperationExecutionResult | undefined = graph.resultByOperation.get(operation); + expect(postRecord?.status).toBe(OperationStatus.Ready); + expect(graph.status).toBe(OperationStatus.Ready); + expect(invalidateCalls.length).toBe(1); + const invalidatedOps: Operation[] = Array.from(invalidateCalls[0].ops as Set); + expect(invalidatedOps).toHaveLength(1); + expect(invalidatedOps[0]).toBe(operation); + expect(invalidateCalls[0].reason).toBe('unit-test'); + }); + + it('invalidates all operations when no iterable is provided', async () => { + const graphOptions: IOperationGraphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + + const op1Runner: IOperationRunner = new MockOperationRunner('op1'); + const op2Runner: IOperationRunner = new MockOperationRunner('op2'); + + const op1: Operation = new Operation({ + runner: op1Runner, + logFilenameIdentifier: 'op1', + phase: mockPhase, + project: getOrCreateProject('p1') + }); + const op2: Operation = new Operation({ + runner: op2Runner, + logFilenameIdentifier: 'op2', + phase: mockPhase, + project: getOrCreateProject('p2') + }); + + const graph: OperationGraph = new OperationGraph(new Set([op1, op2]), graphOptions); + + await graph.executeAsync({}); + for (const record of graph.resultByOperation.values()) { + expect(record.status).toBeDefined(); + } + + graph.invalidateOperations(undefined, 'bulk'); + for (const record of graph.resultByOperation.values()) { + expect(record.status).toBe(OperationStatus.Ready); + } + }); +}); + +describe('deferred invalidation during active iteration', () => { + const deferGraphOptions: IOperationGraphOptions = { + quietMode: false, + debugMode: false, + parallelism: 2, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + + function createNamedOp(name: string, runner?: IOperationRunner): Operation { + return new Operation({ + runner: runner ?? new MockOperationRunner(name, async () => OperationStatus.Success), + phase: mockPhase, + project: getOrCreateProject(name), + logFilenameIdentifier: name + }); + } + + it('does not mutate a completed current-iteration record until after the iteration ends', async () => { + // op2 depends on op1, so op1 completes first and its record is written to resultByOperation. + // The afterExecuteOperationAsync hook for op2 fires while the iteration is still active, giving + // us a natural point to invalidate op1 and verify the deferred path. + const op1: Operation = createNamedOp('defer-op1'); + const op2: Operation = createNamedOp('defer-op2'); + op2.addDependency(op1); + + let op1StatusAtHookTime: OperationStatus | undefined; + let op1StatusAfterInvalidateCall: OperationStatus | undefined; + + const invalidateCalls: Array<{ ops: Operation[]; reason: string | undefined }> = []; + const graph: OperationGraph = new OperationGraph(new Set([op1, op2]), { + ...deferGraphOptions, + abortController: new AbortController() + }); + graph.hooks.afterExecuteOperationAsync.tapPromise('test', async (record) => { + if (record.operation === op2) { + // op1 is already in resultByOperation (set during op1's completion handler) + op1StatusAtHookTime = graph.resultByOperation.get(op1)?.status; + graph.invalidateOperations([op1], 'mid-iter'); + // Deferred — record must not have been mutated + op1StatusAfterInvalidateCall = graph.resultByOperation.get(op1)?.status; + } + }); + graph.hooks.onInvalidateOperations.tap('test', (ops, reason) => { + invalidateCalls.push({ ops: [...(ops as Set)], reason }); + }); + + const result: IExecutionResult = await graph.executeAsync({}); + + // Both ops executed successfully; the abort triggered by invalidation had nothing left to abort. + expect(result.status).toBe(OperationStatus.Success); + + // op1 was in resultByOperation (Success) when the hook fired for op2 + expect(op1StatusAtHookTime).toBe(OperationStatus.Success); + // The deferred path must not mutate the record mid-iteration + expect(op1StatusAfterInvalidateCall).toBe(OperationStatus.Success); + + // After the iteration .finally() fires, the deferred reset is applied + expect(graph.resultByOperation.get(op1)?.status).toBe(OperationStatus.Ready); + + // onInvalidateOperations fires once — after the iteration ends — with [op1] and the reason. + // The synchronous call inside invalidateOperations() is skipped because the invalidated set is empty. + expect(invalidateCalls).toHaveLength(1); + expect(invalidateCalls[0].ops).toHaveLength(1); + expect(invalidateCalls[0].ops[0]).toBe(op1); + expect(invalidateCalls[0].reason).toBe('mid-iter'); + }); + + it('coalesces deferred invalidations with the same reason into a single hook call', async () => { + // op3 depends on both op1 and op2, so both complete before op3 starts. + // The afterExecuteOperationAsync hook for op3 invalidates both with the same reason. + // The deferred .finally() handler must fire a single onInvalidateOperations call for both. + const op1: Operation = createNamedOp('coalesce-op1'); + const op2: Operation = createNamedOp('coalesce-op2'); + const op3: Operation = createNamedOp('coalesce-op3'); + op3.addDependency(op1); + op3.addDependency(op2); + + const graph: OperationGraph = new OperationGraph(new Set([op1, op2, op3]), { + ...deferGraphOptions, + abortController: new AbortController() + }); + graph.hooks.afterExecuteOperationAsync.tapPromise('test', async (record) => { + if (record.operation === op3) { + graph.invalidateOperations([op1], 'same-reason'); + graph.invalidateOperations([op2], 'same-reason'); + } + }); + + const deferredCalls: Array<{ ops: Operation[]; reason: string | undefined }> = []; + graph.hooks.onInvalidateOperations.tap('test', (ops, reason) => { + const opsArray: Operation[] = [...(ops as Set)]; + if (opsArray.length > 0) { + deferredCalls.push({ ops: opsArray, reason }); + } + }); + + await graph.executeAsync({}); + + // Only one deferred call: both op1 and op2 arrive together under 'same-reason' + expect(deferredCalls).toHaveLength(1); + expect(deferredCalls[0].reason).toBe('same-reason'); + expect(new Set(deferredCalls[0].ops)).toEqual(new Set([op1, op2])); + + // Both records are now Ready + expect(graph.resultByOperation.get(op1)?.status).toBe(OperationStatus.Ready); + expect(graph.resultByOperation.get(op2)?.status).toBe(OperationStatus.Ready); + }); + + it('fires separate hook calls for deferred invalidations with distinct reasons', async () => { + const op1: Operation = createNamedOp('distinct-op1'); + const op2: Operation = createNamedOp('distinct-op2'); + const op3: Operation = createNamedOp('distinct-op3'); + op3.addDependency(op1); + op3.addDependency(op2); + + const graph: OperationGraph = new OperationGraph(new Set([op1, op2, op3]), { + ...deferGraphOptions, + abortController: new AbortController() + }); + graph.hooks.afterExecuteOperationAsync.tapPromise('test', async (record) => { + if (record.operation === op3) { + graph.invalidateOperations([op1], 'reason-a'); + graph.invalidateOperations([op2], 'reason-b'); + } + }); + + const deferredCalls: Array<{ ops: Operation[]; reason: string | undefined }> = []; + graph.hooks.onInvalidateOperations.tap('test', (ops, reason) => { + const opsArray: Operation[] = [...(ops as Set)]; + if (opsArray.length > 0) { + deferredCalls.push({ ops: opsArray, reason }); + } + }); + + await graph.executeAsync({}); + + // Two separate deferred calls — one per reason + expect(deferredCalls).toHaveLength(2); + const callA: { ops: Operation[]; reason: string | undefined } | undefined = deferredCalls.find( + (c) => c.reason === 'reason-a' + ); + const callB: { ops: Operation[]; reason: string | undefined } | undefined = deferredCalls.find( + (c) => c.reason === 'reason-b' + ); + expect(callA?.ops).toHaveLength(1); + expect(callA?.ops[0]).toBe(op1); + expect(callB?.ops).toHaveLength(1); + expect(callB?.ops[0]).toBe(op2); + }); + + it('skips operations that are already in a non-terminal (Ready) state', async () => { + // Run the graph, then manually invalidate an op to put it in Ready state. + // A second invalidateOperations call on the same (already-Ready) op must be a no-op: + // TERMINAL_STATUSES does not include Ready, so the guard prevents processing. + const op: Operation = createNamedOp('skip-ready-op'); + const graph: OperationGraph = new OperationGraph(new Set([op]), { + ...deferGraphOptions, + abortController: new AbortController() + }); + + await graph.executeAsync({}); + expect(graph.resultByOperation.get(op)?.status).toBe(OperationStatus.Success); + + // First invalidation: Success → Ready + graph.invalidateOperations([op], 'first'); + expect(graph.resultByOperation.get(op)?.status).toBe(OperationStatus.Ready); + + // Now tap AFTER the first invalidation so we only observe the second call + const secondCallOps: Operation[][] = []; + graph.hooks.onInvalidateOperations.tap('test-second', (ops) => { + secondCallOps.push([...(ops as Set)]); + }); + + // Second invalidation: op is in Ready state (not terminal) — should be skipped + graph.invalidateOperations([op], 'second'); + + // Hook is not called at all — the invalidated set was empty, so the call is skipped + expect(secondCallOps).toHaveLength(0); + expect(graph.resultByOperation.get(op)?.status).toBe(OperationStatus.Ready); // unchanged + }); +}); + +describe('closeRunnersAsync', () => { + class ClosableRunner extends MockOperationRunner { + public readonly closeAsync: jest.Mock, []> = jest.fn(async () => { + /* no-op */ + }); + } + + it('invokes closeAsync on runners and triggers onExecutionStatesUpdated hook', async () => { + const localOptions: IOperationGraphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + + const runner = new ClosableRunner('closable'); + const graph: OperationGraph = createGraph(localOptions, runner); + + await graph.executeAsync({}); + + const statusChangedCalls: ReadonlySet[] = []; + graph.hooks.onExecutionStatesUpdated.tap('test', (records) => { + statusChangedCalls.push(records); + }); + + await graph.closeRunnersAsync(); + + expect(runner.closeAsync).toHaveBeenCalledTimes(1); + expect(statusChangedCalls.length).toBe(1); + const firstBatchArray = Array.from(statusChangedCalls[0]); + expect(firstBatchArray[0].operation.runner).toBe(runner); + }); + + it('only closes specified runners when operations iterable provided', async () => { + const graphOptions: IOperationGraphOptions = { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + + const runner1 = new ClosableRunner('closable1'); + const runner2 = new ClosableRunner('closable2'); + + const op1: Operation = new Operation({ + runner: runner1, + logFilenameIdentifier: 'c1', + phase: mockPhase, + project: getOrCreateProject('c1') + }); + const op2: Operation = new Operation({ + runner: runner2, + logFilenameIdentifier: 'c2', + phase: mockPhase, + project: getOrCreateProject('c2') + }); + + const graph: OperationGraph = new OperationGraph(new Set([op1, op2]), graphOptions); + await graph.executeAsync({}); + + await graph.closeRunnersAsync([op1]); + expect(runner1.closeAsync).toHaveBeenCalledTimes(1); + expect(runner2.closeAsync).not.toHaveBeenCalled(); + }); +}); + +describe('Graph state change notifications', () => { + function createGraphForStateTests(overrides: Partial = {}): OperationGraph { + const baseOptions: IOperationGraphOptions = { + quietMode: false, + debugMode: false, + parallelism: 2, + maxParallelism: 4, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }; + return new OperationGraph(new Set(), { ...baseOptions, ...overrides }); + } + + beforeEach(() => { + jest.useFakeTimers({ legacyFakeTimers: true }); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + async function flushNextTick(): Promise { + jest.runAllTicks(); + } + + it('invokes callback when a single property changes', async () => { + const graph: OperationGraph = createGraphForStateTests(); + const calls: IOperationGraph[] = []; + graph.hooks.onGraphStateChanged.tap('test', (m) => calls.push(m)); + graph.debugMode = true; + expect(calls.length).toBe(0); + await flushNextTick(); + expect(calls.length).toBe(1); + expect(graph.debugMode).toBe(true); + }); + + it('debounces multiple property changes in the same tick', async () => { + const graph: OperationGraph = createGraphForStateTests(); + const calls: IOperationGraph[] = []; + graph.hooks.onGraphStateChanged.tap('test', (m) => calls.push(m)); + graph.debugMode = true; + graph.quietMode = true; + graph.pauseNextIteration = true; + graph.parallelism = 3; + await flushNextTick(); + expect(calls.length).toBe(1); + expect(graph.debugMode).toBe(true); + expect(graph.quietMode).toBe(true); + expect(graph.pauseNextIteration).toBe(true); + expect(graph.parallelism).toBe(3); + }); + + it('does not invoke callback when setting a property to its existing value', async () => { + const graph: OperationGraph = createGraphForStateTests(); + const calls: IOperationGraph[] = []; + graph.hooks.onGraphStateChanged.tap('test', (m) => calls.push(m)); + graph.debugMode = false; + graph.quietMode = false; + await flushNextTick(); + expect(calls.length).toBe(0); + }); + + it('clamps parallelism to configured bounds and invokes callback only when value changes', async () => { + const graph: OperationGraph = createGraphForStateTests({ + parallelism: 2, + maxParallelism: 4 + }); + const calls: IOperationGraph[] = []; + graph.hooks.onGraphStateChanged.tap('test', (m) => calls.push(m)); + + // Increase beyond max -> clamp to 4 + graph.parallelism = 10; + await flushNextTick(); + expect(graph.parallelism).toBe(4); + expect(calls.length).toBe(1); + + // Set to same clamped value -> no new callback + graph.parallelism = 10; // still clamps to 4, unchanged + await flushNextTick(); + expect(calls.length).toBe(1); + + // Decrease below minimum -> clamp to 1 (change from 4) + graph.parallelism = 0; + await flushNextTick(); + expect(graph.parallelism).toBe(1); + expect(calls.length).toBe(2); + }); + + it('pauseNextIteration change triggers callback only when value changes', async () => { + const graph: OperationGraph = createGraphForStateTests(); + const calls: IOperationGraph[] = []; + graph.hooks.onGraphStateChanged.tap('test', (m) => calls.push(m)); + + graph.pauseNextIteration = true; + await flushNextTick(); + expect(calls.length).toBe(1); + expect(graph.pauseNextIteration).toBe(true); + + graph.pauseNextIteration = true; + await flushNextTick(); + expect(calls.length).toBe(1); + + graph.pauseNextIteration = false; + await flushNextTick(); + expect(calls.length).toBe(2); + expect(graph.pauseNextIteration).toBe(false); + }); +}); + +describe('setEnabledStates', () => { + function createChain(names: string[]): Operation[] { + const ops: Operation[] = names.map( + (n) => + new Operation({ + runner: new MockOperationRunner(n, async () => OperationStatus.Success), + phase: mockPhase, + project: getOrCreateProject(n), + logFilenameIdentifier: n + }) + ); + // Simple linear dependencies a->b->c (each depends on next) for dependency expansion tests + for (let i = 0; i < ops.length - 1; i++) { + ops[i].addDependency(ops[i + 1]); + } + return ops; + } + + function createGraphWithOperations(ops: Operation[]): OperationGraph { + return new OperationGraph(new Set(ops), { + quietMode: false, + debugMode: false, + parallelism: 1, + allowOversubscription: true, + destinations: [mockWritable], + abortController: new AbortController() + }); + } + + it('safe enable expands dependencies', () => { + const [a, b, c] = createChain(['a', 'b', 'c']); + // start disabled + a.enabled = false; + b.enabled = false; + c.enabled = false; + const graph = createGraphWithOperations([a, b, c]); + const calls: ReadonlySet[] = []; + graph.hooks.onEnableStatesChanged.tap('test', (ops) => calls.push(new Set(ops))); + const changed = graph.setEnabledStates([a], true, 'safe'); + expect(changed).toBe(true); + // All three should now be true because of dependency expansion (a depends on b depends on c) + expect(a.enabled).toBe(true); + expect(b.enabled).toBe(true); + expect(c.enabled).toBe(true); + expect(calls).toHaveLength(1); + expect(Array.from(calls[0]).sort((x, y) => x.name!.localeCompare(y.name!))).toEqual([a, b, c]); + }); + + it('safe disable disables entire dependency subtree when not required elsewhere', () => { + const [a, b, c] = createChain(['a', 'b', 'c']); + // Initially all true + const graph = createGraphWithOperations([a, b, c]); + const calls: ReadonlySet[] = []; + graph.hooks.onEnableStatesChanged.tap('test', (ops) => calls.push(new Set(ops))); + // Attempt to disable middle dependency (b) safely -> should NOT disable since a depends on b (b and its subtree still required) + const changedB = graph.setEnabledStates([b], false, 'safe'); + expect(changedB).toBe(false); + expect(calls).toHaveLength(0); + // Attempt to disable leaf c safely -> should NOT disable since b (and thus a) still depend on c + const changedC = graph.setEnabledStates([c], false, 'safe'); + expect(changedC).toBe(false); + expect(calls).toHaveLength(0); + // Disable root a safely -> this should disable a and its entire dependency subtree (b,c) since nothing else depends on them + const changedA = graph.setEnabledStates([a], false, 'safe'); + expect(changedA).toBe(true); + expect(a.enabled).toBe(false); + expect(b.enabled).toBe(false); + expect(c.enabled).toBe(false); + expect(calls).toHaveLength(1); // single batch for subtree disable + const changedNames: string[] = Array.from(calls[0], (op) => op.name!).sort(); + expect(changedNames).toEqual(['a', 'b', 'c']); + }); + + it('safe ignore-dependency-changes sets requested and dependencies to ignore state, respects per-op flag', () => { + const [a, b, c] = createChain(['a', 'b', 'c']); + // Simulate b having ignoreChangedProjectsOnlyFlag forcing it to true rather than ignore-dependency-changes + b.settings = { ignoreChangedProjectsOnlyFlag: true } as unknown as typeof b.settings; + a.enabled = false; + b.enabled = false; + c.enabled = false; + const graph = createGraphWithOperations([a, b, c]); + const calls: ReadonlySet[] = []; + graph.hooks.onEnableStatesChanged.tap('test', (ops) => calls.push(new Set(ops))); + const changed = graph.setEnabledStates([a], 'ignore-dependency-changes', 'safe'); + expect(changed).toBe(true); + expect(a.enabled).toBe('ignore-dependency-changes'); + // b forced to true because of its settings flag + expect(b.enabled).toBe(true); + const cState: Operation['enabled'] = c.enabled; + const acceptable: boolean = + cState === (true as Operation['enabled']) || + cState === ('ignore-dependency-changes' as Operation['enabled']); + expect(acceptable).toBe(true); + expect(calls).toHaveLength(1); + // a and b at least must be in changed set (c may also if changed) + const changedNames = new Set(Array.from(calls[0], (o) => o.name)); + expect(changedNames.has('a')).toBe(true); + expect(changedNames.has('b')).toBe(true); + }); + + it('unsafe mode only mutates provided operations', () => { + const [a, b, c] = createChain(['a', 'b', 'c']); + const graph = createGraphWithOperations([a, b, c]); + const calls: ReadonlySet[] = []; + graph.hooks.onEnableStatesChanged.tap('test', (ops) => calls.push(new Set(ops))); + const changed = graph.setEnabledStates([b], false, 'unsafe'); + expect(changed).toBe(true); + expect(a.enabled).not.toBe(false); + expect(b.enabled).toBe(false); + expect(c.enabled).not.toBe(false); + expect(calls).toHaveLength(1); + expect(Array.from(calls[0])).toEqual([b]); + }); +}); diff --git a/libraries/rush-lib/src/logic/operations/test/ParseParallelism.test.ts b/libraries/rush-lib/src/logic/operations/test/ParseParallelism.test.ts new file mode 100644 index 00000000000..c038e2f6771 --- /dev/null +++ b/libraries/rush-lib/src/logic/operations/test/ParseParallelism.test.ts @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { coerceParallelism, parseParallelism } from '../ParseParallelism'; + +describe(parseParallelism.name, () => { + it('throwsErrorOnInvalidParallelism', () => { + expect(() => parseParallelism('tequila')).toThrowErrorMatchingSnapshot(); + }); + + it('throwsErrorOnInvalidParallelismPercentage', () => { + expect(() => parseParallelism('200%')).toThrowErrorMatchingSnapshot(); + }); + + it('returns scalar 1 for "max"', () => { + expect(parseParallelism('max')).toEqual({ scalar: 1 }); + }); + + it('returns a scalar for a percentage input', () => { + expect(parseParallelism('50%')).toEqual({ scalar: 0.5 }); + }); + + it('returns a raw number for a numeric input', () => { + expect(parseParallelism('4')).toBe(4); + }); + + it('trims whitespace from input', () => { + expect(parseParallelism(' 4 ')).toBe(4); + }); +}); + +describe(coerceParallelism.name, () => { + describe('raw numeric values', () => { + it('passes through a number within range', () => { + expect(coerceParallelism(4, 8)).toBe(4); + }); + + it('clamps a number above maxParallelism down to maxParallelism', () => { + expect(coerceParallelism(16, 8)).toBe(8); + }); + + it('clamps a negative number up to 0', () => { + expect(coerceParallelism(-1, 8)).toBe(0); + }); + + it('allows 0', () => { + expect(coerceParallelism(0, 8)).toBe(0); + }); + }); + + describe('scalar values', () => { + it('converts scalar 1 to maxParallelism', () => { + expect(coerceParallelism({ scalar: 1 }, 8)).toBe(8); + }); + + it('converts scalar 0.5 to half of maxParallelism', () => { + expect(coerceParallelism({ scalar: 0.5 }, 8)).toBe(4); + }); + + it('floors fractional results', () => { + // floor(0.333333 * 8) = floor(2.666...) = 2 + expect(coerceParallelism({ scalar: 0.333333 }, 8)).toBe(2); + }); + + it('clamps scalar result to at least 1', () => { + // floor(0.001 * 8) = 0, clamped up to 1 + expect(coerceParallelism({ scalar: 0.001 }, 8)).toBe(1); + }); + + it('Windows default scalar (0.999) yields one less than maxParallelism', () => { + // floor(0.999 * 8) = floor(7.992) = 7 + expect(coerceParallelism({ scalar: 0.999 }, 8)).toBe(7); + }); + }); +}); diff --git a/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts index ef16651af85..109c96676b8 100644 --- a/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/PhasedOperationPlugin.test.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import { JsonFile } from '@rushstack/node-core-library'; import { RushConfiguration } from '../../../api/RushConfiguration'; +import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; import { CommandLineConfiguration, type IPhasedCommandConfig } from '../../../api/CommandLineConfiguration'; import { PhasedOperationPlugin } from '../PhasedOperationPlugin'; import type { Operation } from '../Operation'; @@ -13,8 +14,13 @@ import { RushConstants } from '../../RushConstants'; import { MockOperationRunner } from './MockOperationRunner'; import { type ICreateOperationsContext, + type IOperationGraphContext, PhasedCommandHooks } from '../../../pluginFramework/PhasedCommandHooks'; +import type { IOperationGraph } from '../IOperationGraph'; +import { OperationGraphHooks } from '../../../pluginFramework/OperationGraphHooks'; +type IOperationExecutionManager = IOperationGraph; +type IOperationExecutionManagerContext = IOperationGraphContext; function serializeOperation(operation: Operation): string { return `${operation.name} (${operation.enabled ? 'enabled' : 'disabled'}${operation.runner!.silent ? ', silent' : ''}) -> [${Array.from( @@ -66,61 +72,61 @@ describe(PhasedOperationPlugin.name, () => { } interface ITestCreateOperationsContext { - phaseOriginal?: ICreateOperationsContext['phaseOriginal']; phaseSelection: ICreateOperationsContext['phaseSelection']; projectSelection: ICreateOperationsContext['projectSelection']; - projectsInUnknownState: ICreateOperationsContext['projectsInUnknownState']; includePhaseDeps?: ICreateOperationsContext['includePhaseDeps']; + generateFullGraph?: ICreateOperationsContext['generateFullGraph']; } + let rushConfiguration!: RushConfiguration; + let commandLineConfiguration!: CommandLineConfiguration; + + beforeAll(() => { + rushConfiguration = RushConfiguration.loadFromConfigurationFile(rushJsonFile); + const commandLineJson: ICommandLineJson = JsonFile.load(commandLineJsonFile); + + commandLineConfiguration = new CommandLineConfiguration(commandLineJson); + }); + async function testCreateOperationsAsync(options: ITestCreateOperationsContext): Promise> { - const { - phaseSelection, - projectSelection, - projectsInUnknownState, - phaseOriginal = phaseSelection, - includePhaseDeps = false - } = options; + const { phaseSelection, projectSelection, includePhaseDeps = false, generateFullGraph = false } = options; const hooks: PhasedCommandHooks = new PhasedCommandHooks(); // Apply the plugin being tested new PhasedOperationPlugin().apply(hooks); // Add mock runners for included operations. - hooks.createOperations.tap('MockOperationRunnerPlugin', createMockRunner); - - const context: Pick< - ICreateOperationsContext, - | 'includePhaseDeps' - | 'phaseOriginal' - | 'phaseSelection' - | 'projectSelection' - | 'projectsInUnknownState' - | 'projectConfigurations' - > = { - includePhaseDeps, - phaseOriginal, + hooks.createOperationsAsync.tap('MockOperationRunnerPlugin', createMockRunner); + + const context: Partial = { phaseSelection, projectSelection, - projectsInUnknownState, - projectConfigurations: new Map() + projectConfigurations: new Map(), + includePhaseDeps, + generateFullGraph, + // Minimal required fields for plugin logic not used directly in these tests + changedProjectsOnly: false, + isIncrementalBuildAllowed: true, + isWatch: generateFullGraph, // simulate watch when using full graph flag + customParameters: new Map(), + rushConfiguration }; - const operations: Set = await hooks.createOperations.promise( + const operations: Set = await hooks.createOperationsAsync.promise( new Set(), context as ICreateOperationsContext ); + const executionHooks: OperationGraphHooks = new OperationGraphHooks(); + const executionManager: Partial = { + operations, + hooks: executionHooks + }; + await hooks.onGraphCreatedAsync.promise( + executionManager as IOperationExecutionManager, + context as IOperationExecutionManagerContext + ); + return operations; } - let rushConfiguration!: RushConfiguration; - let commandLineConfiguration!: CommandLineConfiguration; - - beforeAll(() => { - rushConfiguration = RushConfiguration.loadFromConfigurationFile(rushJsonFile); - const commandLineJson: ICommandLineJson = JsonFile.load(commandLineJsonFile); - - commandLineConfiguration = new CommandLineConfiguration(commandLineJson); - }); - it('handles a full build', async () => { const buildCommand: IPhasedCommandConfig = commandLineConfiguration.commands.get( 'build' @@ -128,8 +134,7 @@ describe(PhasedOperationPlugin.name, () => { const operations: Set = await testCreateOperationsAsync({ phaseSelection: buildCommand.phases, - projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set(rushConfiguration.projects) + projectSelection: new Set(rushConfiguration.projects) }); // All projects @@ -143,8 +148,7 @@ describe(PhasedOperationPlugin.name, () => { let operations: Set = await testCreateOperationsAsync({ phaseSelection: buildCommand.phases, - projectSelection: new Set([rushConfiguration.getProjectByName('g')!]), - projectsInUnknownState: new Set([rushConfiguration.getProjectByName('g')!]) + projectSelection: new Set([rushConfiguration.getProjectByName('g')!]) }); // Single project @@ -156,11 +160,6 @@ describe(PhasedOperationPlugin.name, () => { rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]), - projectsInUnknownState: new Set([ - rushConfiguration.getProjectByName('f')!, - rushConfiguration.getProjectByName('a')!, - rushConfiguration.getProjectByName('c')! ]) }); @@ -168,99 +167,31 @@ describe(PhasedOperationPlugin.name, () => { expectOperationsToMatchSnapshot(operations, 'filtered'); }); - it('handles some changed projects', async () => { - const buildCommand: IPhasedCommandConfig = commandLineConfiguration.commands.get( - 'build' - )! as IPhasedCommandConfig; - - let operations: Set = await testCreateOperationsAsync({ - phaseSelection: buildCommand.phases, - projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set([rushConfiguration.getProjectByName('g')!]) - }); - - // Single project - expectOperationsToMatchSnapshot(operations, 'single'); - - operations = await testCreateOperationsAsync({ - phaseSelection: buildCommand.phases, - projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set([ - rushConfiguration.getProjectByName('f')!, - rushConfiguration.getProjectByName('a')!, - rushConfiguration.getProjectByName('c')! - ]) - }); - - // Filtered projects - expectOperationsToMatchSnapshot(operations, 'multiple'); - }); - - it('handles some changed projects within filtered projects', async () => { - const buildCommand: IPhasedCommandConfig = commandLineConfiguration.commands.get( - 'build' - )! as IPhasedCommandConfig; - - const operations: Set = await testCreateOperationsAsync({ - phaseSelection: buildCommand.phases, - projectSelection: new Set([ - rushConfiguration.getProjectByName('f')!, - rushConfiguration.getProjectByName('a')!, - rushConfiguration.getProjectByName('c')! - ]), - projectsInUnknownState: new Set([ - rushConfiguration.getProjectByName('a')!, - rushConfiguration.getProjectByName('c')! - ]) - }); - - // Single project - expectOperationsToMatchSnapshot(operations, 'multiple'); - }); - - it('handles different phaseOriginal vs phaseSelection without --include-phase-deps', async () => { + it('handles incomplete phaseSelection without --include-phase-deps', async () => { const operations: Set = await testCreateOperationsAsync({ includePhaseDeps: false, - phaseSelection: new Set([ - commandLineConfiguration.phases.get('_phase:no-deps')!, - commandLineConfiguration.phases.get('_phase:upstream-self')! - ]), - phaseOriginal: new Set([commandLineConfiguration.phases.get('_phase:upstream-self')!]), - projectSelection: new Set([rushConfiguration.getProjectByName('a')!]), - projectsInUnknownState: new Set([rushConfiguration.getProjectByName('a')!]) + phaseSelection: new Set([commandLineConfiguration.phases.get('_phase:upstream-self')!]), + projectSelection: new Set([rushConfiguration.getProjectByName('a')!]) }); expectOperationsToMatchSnapshot(operations, 'single-project'); }); - it('handles different phaseOriginal vs phaseSelection with --include-phase-deps', async () => { + it('handles incomplete phaseSelection with --include-phase-deps', async () => { const operations: Set = await testCreateOperationsAsync({ includePhaseDeps: true, - phaseSelection: new Set([ - commandLineConfiguration.phases.get('_phase:no-deps')!, - commandLineConfiguration.phases.get('_phase:upstream-self')! - ]), - phaseOriginal: new Set([commandLineConfiguration.phases.get('_phase:upstream-self')!]), - projectSelection: new Set([rushConfiguration.getProjectByName('a')!]), - projectsInUnknownState: new Set([rushConfiguration.getProjectByName('a')!]) + phaseSelection: new Set([commandLineConfiguration.phases.get('_phase:upstream-self')!]), + projectSelection: new Set([rushConfiguration.getProjectByName('a')!]) }); expectOperationsToMatchSnapshot(operations, 'single-project'); }); - it('handles different phaseOriginal vs phaseSelection cross-project with --include-phase-deps', async () => { + it('handles incomplete phaseSelection cross-project with --include-phase-deps', async () => { const operations: Set = await testCreateOperationsAsync({ includePhaseDeps: true, - phaseSelection: new Set([ - commandLineConfiguration.phases.get('_phase:no-deps')!, - commandLineConfiguration.phases.get('_phase:upstream-1')! - ]), - phaseOriginal: new Set([commandLineConfiguration.phases.get('_phase:upstream-1')!]), - projectSelection: new Set([ - rushConfiguration.getProjectByName('a')!, - rushConfiguration.getProjectByName('h')! - ]), - projectsInUnknownState: new Set([rushConfiguration.getProjectByName('h')!]) + phaseSelection: new Set([commandLineConfiguration.phases.get('_phase:upstream-1')!]), + projectSelection: new Set([rushConfiguration.getProjectByName('h')!]) }); expectOperationsToMatchSnapshot(operations, 'multiple-project'); @@ -270,8 +201,7 @@ describe(PhasedOperationPlugin.name, () => { // Single phase with a missing dependency let operations: Set = await testCreateOperationsAsync({ phaseSelection: new Set([commandLineConfiguration.phases.get('_phase:upstream-self')!]), - projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set(rushConfiguration.projects) + projectSelection: new Set(rushConfiguration.projects) }); expectOperationsToMatchSnapshot(operations, 'single-phase'); @@ -283,8 +213,7 @@ describe(PhasedOperationPlugin.name, () => { commandLineConfiguration.phases.get('_phase:upstream-1')!, commandLineConfiguration.phases.get('_phase:no-deps')! ]), - projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set(rushConfiguration.projects) + projectSelection: new Set(rushConfiguration.projects) }); expectOperationsToMatchSnapshot(operations, 'two-phases'); }); @@ -297,11 +226,6 @@ describe(PhasedOperationPlugin.name, () => { rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]), - projectsInUnknownState: new Set([ - rushConfiguration.getProjectByName('f')!, - rushConfiguration.getProjectByName('a')!, - rushConfiguration.getProjectByName('c')! ]) }); expectOperationsToMatchSnapshot(operations, 'single-phase'); @@ -318,13 +242,27 @@ describe(PhasedOperationPlugin.name, () => { rushConfiguration.getProjectByName('f')!, rushConfiguration.getProjectByName('a')!, rushConfiguration.getProjectByName('c')! - ]), - projectsInUnknownState: new Set([ - rushConfiguration.getProjectByName('f')!, - rushConfiguration.getProjectByName('a')!, - rushConfiguration.getProjectByName('c')! ]) }); expectOperationsToMatchSnapshot(operations, 'missing-links'); }); + + it('includes full graph but enables subset when generateFullGraph is true', async () => { + const buildCommand: IPhasedCommandConfig = commandLineConfiguration.commands.get( + 'build' + )! as IPhasedCommandConfig; + const subset: Set = new Set([ + rushConfiguration.getProjectByName('a')!, + rushConfiguration.getProjectByName('c')! + ]); + + const operations: Set = await testCreateOperationsAsync({ + phaseSelection: buildCommand.phases, + projectSelection: subset, + generateFullGraph: true + }); + + // Expect all projects to be present, but only selected subset enabled + expectOperationsToMatchSnapshot(operations, 'full-graph-filtered'); + }); }); diff --git a/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts index 818144e85df..3e558acc906 100644 --- a/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts +++ b/libraries/rush-lib/src/logic/operations/test/ShellOperationRunnerPlugin.test.ts @@ -77,16 +77,10 @@ describe(ShellOperationRunnerPlugin.name, () => { const fakeCreateOperationsContext: Pick< ICreateOperationsContext, - | 'phaseOriginal' - | 'phaseSelection' - | 'projectSelection' - | 'projectsInUnknownState' - | 'projectConfigurations' + 'phaseSelection' | 'projectSelection' | 'projectConfigurations' > = { - phaseOriginal: echoCommand.phases, phaseSelection: echoCommand.phases, projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set(rushConfiguration.projects), projectConfigurations: new Map() }; @@ -97,7 +91,7 @@ describe(ShellOperationRunnerPlugin.name, () => { // Applies the Shell Operation Runner to selected operations new ShellOperationRunnerPlugin().apply(hooks); - const operations: Set = await hooks.createOperations.promise( + const operations: Set = await hooks.createOperationsAsync.promise( new Set(), fakeCreateOperationsContext as ICreateOperationsContext ); @@ -125,16 +119,10 @@ describe(ShellOperationRunnerPlugin.name, () => { const fakeCreateOperationsContext: Pick< ICreateOperationsContext, - | 'phaseOriginal' - | 'phaseSelection' - | 'projectSelection' - | 'projectsInUnknownState' - | 'projectConfigurations' + 'phaseSelection' | 'projectSelection' | 'projectConfigurations' > = { - phaseOriginal: echoCommand.phases, phaseSelection: echoCommand.phases, projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set(rushConfiguration.projects), projectConfigurations: new Map() }; @@ -145,7 +133,7 @@ describe(ShellOperationRunnerPlugin.name, () => { // Applies the Shell Operation Runner to selected operations new ShellOperationRunnerPlugin().apply(hooks); - const operations: Set = await hooks.createOperations.promise( + const operations: Set = await hooks.createOperationsAsync.promise( new Set(), fakeCreateOperationsContext as ICreateOperationsContext ); @@ -221,18 +209,14 @@ describe(ShellOperationRunnerPlugin.name, () => { const fakeCreateOperationsContext: Pick< ICreateOperationsContext, - | 'phaseOriginal' | 'phaseSelection' | 'projectSelection' - | 'projectsInUnknownState' | 'projectConfigurations' | 'rushConfiguration' | 'customParameters' > = { - phaseOriginal: buildCommand.phases, phaseSelection: buildCommand.phases, projectSelection: new Set(rushConfiguration.projects), - projectsInUnknownState: new Set(rushConfiguration.projects), projectConfigurations, rushConfiguration, customParameters: customParametersForContext @@ -245,9 +229,9 @@ describe(ShellOperationRunnerPlugin.name, () => { // Applies the Shell Operation Runner to selected operations new ShellOperationRunnerPlugin().apply(hooks); - const operations: Set = await hooks.createOperations.promise( + const operations: Set = await hooks.createOperationsAsync.promise( new Set(), - fakeCreateOperationsContext as ICreateOperationsContext + fakeCreateOperationsContext as unknown as ICreateOperationsContext ); // Verify that project 'a' has the --production parameter filtered out diff --git a/libraries/rush-lib/src/logic/operations/test/WeightedOperationPlugin.test.ts b/libraries/rush-lib/src/logic/operations/test/WeightedOperationPlugin.test.ts deleted file mode 100644 index 7aec8361ede..00000000000 --- a/libraries/rush-lib/src/logic/operations/test/WeightedOperationPlugin.test.ts +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. -// See LICENSE in the project root for license information. - -import os from 'node:os'; - -import type { IPhase } from '../../../api/CommandLineConfiguration'; -import type { RushConfigurationProject } from '../../../api/RushConfigurationProject'; -import type { IOperationSettings, RushProjectConfiguration } from '../../../api/RushProjectConfiguration'; -import { - type IExecuteOperationsContext, - PhasedCommandHooks -} from '../../../pluginFramework/PhasedCommandHooks'; -import type { IOperationExecutionResult } from '../IOperationExecutionResult'; -import { Operation } from '../Operation'; -import { WeightedOperationPlugin } from '../WeightedOperationPlugin'; -import { MockOperationRunner } from './MockOperationRunner'; - -const MOCK_PHASE: IPhase = { - name: '_phase:test', - allowWarningsOnSuccess: false, - associatedParameters: new Set(), - dependencies: { - self: new Set(), - upstream: new Set() - }, - isSynthetic: false, - logFilenameIdentifier: '_phase_test', - missingScriptBehavior: 'silent' -}; - -function createProject(packageName: string): RushConfigurationProject { - return { - packageName - } as RushConfigurationProject; -} - -function createOperation(options: { - project: RushConfigurationProject; - settings?: IOperationSettings; - isNoOp?: boolean; -}): Operation { - const { project, settings, isNoOp } = options; - return new Operation({ - phase: MOCK_PHASE, - project, - settings, - runner: new MockOperationRunner(`${project.packageName} (${MOCK_PHASE.name})`, undefined, false, isNoOp), - logFilenameIdentifier: `${project.packageName}_phase_test` - }); -} - -function createExecutionRecords(operation: Operation): Map { - return new Map([ - [ - operation, - { - operation, - runner: operation.runner - } as unknown as IOperationExecutionResult - ] - ]); -} - -function createContext( - projectConfigurations: ReadonlyMap, - parallelism: number = os.availableParallelism() -): IExecuteOperationsContext { - return { - projectConfigurations, - parallelism - } as IExecuteOperationsContext; -} - -async function applyWeightPluginAsync( - operations: Map, - context: IExecuteOperationsContext -): Promise { - const hooks: PhasedCommandHooks = new PhasedCommandHooks(); - new WeightedOperationPlugin().apply(hooks); - await hooks.beforeExecuteOperations.promise(operations, context); -} - -function mockAvailableParallelism(value: number): jest.SpyInstance { - return jest.spyOn(os, 'availableParallelism').mockReturnValue(value); -} - -describe(WeightedOperationPlugin.name, () => { - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('applies numeric weight from operation settings', async () => { - const project: RushConfigurationProject = createProject('project-number'); - const operation: Operation = createOperation({ - project, - settings: { - operationName: MOCK_PHASE.name, - weight: 7 - } - }); - - await applyWeightPluginAsync( - createExecutionRecords(operation), - createContext(new Map(), /* Set parallelism to ensure -p does not affect weight calculation */ 1) - ); - - expect(operation.weight).toBe(7); - }); - - it('converts percentage weight using available parallelism', async () => { - mockAvailableParallelism(10); - - const project: RushConfigurationProject = createProject('project-percent'); - const operation: Operation = createOperation({ - project, - settings: { - operationName: MOCK_PHASE.name, - weight: '25%' - } as IOperationSettings - }); - - await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map())); - - expect(operation.weight).toBe(2); - }); - - it('reads weight from rush-project configuration when operation settings are undefined', async () => { - mockAvailableParallelism(8); - - const project: RushConfigurationProject = createProject('project-config'); - const operation: Operation = createOperation({ project }); - const projectConfiguration: RushProjectConfiguration = { - operationSettingsByOperationName: new Map([ - [ - MOCK_PHASE.name, - { - operationName: MOCK_PHASE.name, - weight: '50%' - } as IOperationSettings - ] - ]) - } as unknown as RushProjectConfiguration; - - await applyWeightPluginAsync( - createExecutionRecords(operation), - createContext(new Map([[project, projectConfiguration]])) - ); - - expect(operation.weight).toBe(4); - }); - - it('use floor when converting percentage weight to avoid zero weight', async () => { - mockAvailableParallelism(16); - - const project: RushConfigurationProject = createProject('project-floor'); - const operation: Operation = createOperation({ - project, - settings: { - operationName: MOCK_PHASE.name, - weight: '33.3333%' - } as IOperationSettings - }); - - await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map())); - - expect(operation.weight).toBe(5); - }); - - it('forces NO-OP operation weight to zero ignore weight settings', async () => { - const project: RushConfigurationProject = createProject('project-no-op'); - const operation: Operation = createOperation({ - project, - isNoOp: true, - settings: { - operationName: MOCK_PHASE.name, - weight: 100 - } - }); - - await applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map())); - - expect(operation.weight).toBe(0); - }); - - it('throws for invalid percentage weight format', async () => { - mockAvailableParallelism(16); - - const project: RushConfigurationProject = createProject('project-invalid'); - const operation: Operation = createOperation({ - project, - // @ts-expect-error Testing invalid input - settings: { - operationName: MOCK_PHASE.name, - weight: '12.5a%' - } as IOperationSettings - }); - - await expect( - applyWeightPluginAsync(createExecutionRecords(operation), createContext(new Map())) - ).rejects.toThrow(/invalid weight: "12.5a%"/i); - }); -}); diff --git a/libraries/rush-lib/src/logic/operations/test/__snapshots__/BuildPlanPlugin.test.ts.snap b/libraries/rush-lib/src/logic/operations/test/__snapshots__/BuildPlanPlugin.test.ts.snap index 619360abed0..22bca251efd 100644 --- a/libraries/rush-lib/src/logic/operations/test/__snapshots__/BuildPlanPlugin.test.ts.snap +++ b/libraries/rush-lib/src/logic/operations/test/__snapshots__/BuildPlanPlugin.test.ts.snap @@ -6,7 +6,11 @@ Array [ "[ log] Build Plan Width (maximum parallelism): 38", "[ log] Number of Nodes per Depth: 22, 38, 33, 11, 1", "[ log] Plan @ Depth 0 has 22 nodes and 0 dependents:", + "[ log] - a (upstream-3)", "[ log] - a (no-deps)", + "[ log] - a (upstream-1)", + "[ log] - a (upstream-1-self-upstream)", + "[ log] - a (upstream-2)", "[ log] - b (no-deps)", "[ log] - c (no-deps)", "[ log] - d (no-deps)", @@ -14,59 +18,55 @@ Array [ "[ log] - f (no-deps)", "[ log] - g (no-deps)", "[ log] - h (no-deps)", - "[ log] - a (upstream-1)", - "[ log] - a (upstream-2)", - "[ log] - a (upstream-1-self-upstream)", + "[ log] - i (upstream-3)", "[ log] - i (no-deps)", - "[ log] - j (no-deps)", "[ log] - i (upstream-1)", - "[ log] - j (upstream-1)", + "[ log] - i (upstream-1-self-upstream)", "[ log] - i (upstream-2)", - "[ log] - j (upstream-2)", - "[ log] - a (upstream-3)", - "[ log] - i (upstream-3)", "[ log] - j (upstream-3)", - "[ log] - i (upstream-1-self-upstream)", + "[ log] - j (no-deps)", + "[ log] - j (upstream-1)", "[ log] - j (upstream-1-self-upstream)", + "[ log] - j (upstream-2)", "[ log] Plan @ Depth 1 has 38 nodes and 22 dependents:", + "[ log] - a (complex)", "[ log] - a (upstream-self)", "[ log] - b (upstream-1)", "[ log] - f (upstream-1)", "[ log] - g (upstream-1)", "[ log] - h (upstream-1)", - "[ log] - b (upstream-self)", - "[ log] - c (upstream-1)", - "[ log] - d (upstream-1)", - "[ log] - c (upstream-self)", - "[ log] - e (upstream-1)", - "[ log] - d (upstream-self)", - "[ log] - e (upstream-self)", - "[ log] - f (upstream-self)", - "[ log] - g (upstream-self)", - "[ log] - h (upstream-self)", "[ log] - b (upstream-2)", "[ log] - f (upstream-2)", "[ log] - g (upstream-2)", "[ log] - h (upstream-2)", "[ log] - a (upstream-1-self)", + "[ log] - b (complex)", + "[ log] - f (complex)", + "[ log] - g (complex)", + "[ log] - h (complex)", "[ log] - b (upstream-3)", "[ log] - f (upstream-3)", "[ log] - g (upstream-3)", "[ log] - h (upstream-3)", "[ log] - a (upstream-2-self)", - "[ log] - b (complex)", - "[ log] - f (complex)", - "[ log] - g (complex)", - "[ log] - h (complex)", + "[ log] - b (upstream-self)", + "[ log] - c (upstream-1)", + "[ log] - d (upstream-1)", + "[ log] - c (upstream-self)", + "[ log] - e (upstream-1)", + "[ log] - d (upstream-self)", + "[ log] - e (upstream-self)", + "[ log] - f (upstream-self)", + "[ log] - g (upstream-self)", + "[ log] - h (upstream-self)", + "[ log] - i (complex)", "[ log] - i (upstream-self)", - "[ log] - j (upstream-self)", "[ log] - i (upstream-1-self)", - "[ log] - j (upstream-1-self)", "[ log] - i (upstream-2-self)", - "[ log] - j (upstream-2-self)", - "[ log] - a (complex)", - "[ log] - i (complex)", "[ log] - j (complex)", + "[ log] - j (upstream-self)", + "[ log] - j (upstream-1-self)", + "[ log] - j (upstream-2-self)", "[ log] Plan @ Depth 2 has 33 nodes and 60 dependents:", "[ log] - b (upstream-self)", "[ log] - f (upstream-self)", @@ -79,13 +79,6 @@ Array [ "[ log] - g (upstream-1-self)", "[ log] - f (upstream-2)", "[ log] - h (upstream-1-self)", - "[ log] - c (upstream-self)", - "[ log] - d (upstream-self)", - "[ log] - e (upstream-2)", - "[ log] - c (upstream-1-self)", - "[ log] - d (upstream-1-self)", - "[ log] - e (upstream-self)", - "[ log] - e (upstream-1-self)", "[ log] - c (upstream-3)", "[ log] - d (upstream-3)", "[ log] - b (upstream-2-self)", @@ -101,6 +94,13 @@ Array [ "[ log] - f (complex)", "[ log] - g (complex)", "[ log] - h (complex)", + "[ log] - c (upstream-self)", + "[ log] - d (upstream-self)", + "[ log] - e (upstream-2)", + "[ log] - c (upstream-1-self)", + "[ log] - d (upstream-1-self)", + "[ log] - e (upstream-self)", + "[ log] - e (upstream-1-self)", "[ log] Plan @ Depth 3 has 11 nodes and 93 dependents:", "[ log] - e (upstream-3)", "[ log] - c (upstream-2-self)", @@ -108,218 +108,218 @@ Array [ "[ log] - c (upstream-1-self-upstream)", "[ log] - d (upstream-1-self-upstream)", "[ log] - f (upstream-1-self-upstream)", - "[ log] - e (upstream-2-self)", - "[ log] - e (upstream-1-self-upstream)", "[ log] - c (complex)", "[ log] - d (complex)", "[ log] - f (complex)", + "[ log] - e (upstream-2-self)", + "[ log] - e (upstream-1-self-upstream)", "[ log] Plan @ Depth 4 has 1 nodes and 104 dependents:", "[ log] - e (complex)", "[ log] ##################################################", - "[ log] a (no-deps): (0)", - "[ log] b (no-deps): (0)", - "[ log] c (no-deps): (0)", - "[ log] d (no-deps): (0)", - "[ log] e (no-deps): (0)", - "[ log] f (no-deps): (0)", - "[ log] g (no-deps): (0)", - "[ log] h (no-deps): (0)", - "[ log] a (upstream-1): (0)", - "[ log] a (upstream-2): (0)", - "[ log] a (upstream-1-self-upstream): (0)", - "[ log] i (no-deps): (1)", - "[ log] j (no-deps): (2)", - "[ log] i (upstream-1): (3)", - "[ log] j (upstream-1): (4)", - "[ log] i (upstream-2): (5)", - "[ log] j (upstream-2): (6)", - "[ log] a (upstream-3): (7)", - "[ log] i (upstream-3): (8)", - "[ log] j (upstream-3): (9)", - "[ log] i (upstream-1-self-upstream): (10)", - "[ log] j (upstream-1-self-upstream): (11)", - "[ log] a (upstream-self): -(0)", - "[ log] b (upstream-1): -(0)", - "[ log] c (upstream-1): -(0)", - "[ log] d (upstream-1): -(0)", - "[ log] e (upstream-1): -(0)", - "[ log] f (upstream-1): -(0)", - "[ log] g (upstream-1): -(0)", - "[ log] h (upstream-1): -(0)", - "[ log] b (upstream-2): -(0)", - "[ log] g (upstream-2): -(0)", - "[ log] h (upstream-2): -(0)", - "[ log] b (upstream-3): -(0)", - "[ log] g (upstream-3): -(0)", - "[ log] h (upstream-3): -(0)", - "[ log] a (upstream-1-self): -(0)", - "[ log] a (upstream-2-self): -(0)", - "[ log] i (upstream-self): -(1)", - "[ log] j (upstream-self): -(2)", - "[ log] i (upstream-1-self): -(3)", - "[ log] j (upstream-1-self): -(4)", - "[ log] i (upstream-2-self): -(5)", - "[ log] j (upstream-2-self): -(6)", - "[ log] a (complex): -(7)", - "[ log] i (complex): -(8)", - "[ log] j (complex): -(9)", - "[ log] b (upstream-self): --(0)", - "[ log] f (upstream-self): --(0)", - "[ log] h (upstream-self): --(0)", - "[ log] g (upstream-self): --(0)", - "[ log] c (upstream-2): --(0)", - "[ log] d (upstream-2): --(0)", - "[ log] e (upstream-2): --(0)", - "[ log] f (upstream-2): --(0)", - "[ log] c (upstream-3): --(0)", - "[ log] d (upstream-3): --(0)", - "[ log] f (upstream-3): --(0)", - "[ log] b (upstream-1-self): --(0)", - "[ log] c (upstream-1-self): --(0)", - "[ log] d (upstream-1-self): --(0)", - "[ log] e (upstream-1-self): --(0)", - "[ log] f (upstream-1-self): --(0)", - "[ log] g (upstream-1-self): --(0)", - "[ log] h (upstream-1-self): --(0)", - "[ log] b (upstream-2-self): --(0)", - "[ log] g (upstream-2-self): --(0)", - "[ log] h (upstream-2-self): --(0)", - "[ log] b (upstream-1-self-upstream): --(0)", - "[ log] g (upstream-1-self-upstream): --(0)", - "[ log] h (upstream-1-self-upstream): --(0)", - "[ log] b (complex): --(0)", - "[ log] g (complex): --(0)", - "[ log] h (complex): --(0)", - "[ log] c (upstream-self): ---(0)", - "[ log] d (upstream-self): ---(0)", - "[ log] e (upstream-3): ---(0)", - "[ log] c (upstream-2-self): ---(0)", - "[ log] d (upstream-2-self): ---(0)", - "[ log] e (upstream-2-self): ---(0)", - "[ log] f (upstream-2-self): ---(0)", - "[ log] c (upstream-1-self-upstream): ---(0)", - "[ log] d (upstream-1-self-upstream): ---(0)", - "[ log] e (upstream-1-self-upstream): ---(0)", - "[ log] f (upstream-1-self-upstream): ---(0)", - "[ log] c (complex): ---(0)", - "[ log] d (complex): ---(0)", - "[ log] f (complex): ---(0)", - "[ log] e (upstream-self): ----(0)", - "[ log] e (complex): ----(0)", + "[ log] a (complex): (0)", + "[ log] a (upstream-3): (0)", + "[ log] a (no-deps): (1)", + "[ log] a (upstream-1): (1)", + "[ log] a (upstream-1-self-upstream): (1)", + "[ log] a (upstream-2): (1)", + "[ log] b (no-deps): (1)", + "[ log] c (no-deps): (1)", + "[ log] d (no-deps): (1)", + "[ log] e (no-deps): (1)", + "[ log] f (no-deps): (1)", + "[ log] g (no-deps): (1)", + "[ log] h (no-deps): (1)", + "[ log] i (complex): (2)", + "[ log] i (upstream-3): (2)", + "[ log] i (no-deps): (3)", + "[ log] i (upstream-1): (4)", + "[ log] i (upstream-1-self-upstream): (5)", + "[ log] i (upstream-2): (6)", + "[ log] j (complex): (7)", + "[ log] j (upstream-3): (7)", + "[ log] j (no-deps): (8)", + "[ log] j (upstream-1): (9)", + "[ log] j (upstream-1-self-upstream): (10)", + "[ log] j (upstream-2): (11)", + "[ log] a (upstream-1-self): -(1)", + "[ log] a (upstream-2-self): -(1)", + "[ log] a (upstream-self): -(1)", + "[ log] b (upstream-1): -(1)", + "[ log] b (upstream-2): -(1)", + "[ log] b (upstream-3): -(1)", + "[ log] c (upstream-1): -(1)", + "[ log] d (upstream-1): -(1)", + "[ log] e (upstream-1): -(1)", + "[ log] f (upstream-1): -(1)", + "[ log] f (upstream-2): -(1)", + "[ log] f (upstream-3): -(1)", + "[ log] g (upstream-1): -(1)", + "[ log] g (upstream-2): -(1)", + "[ log] g (upstream-3): -(1)", + "[ log] h (upstream-1): -(1)", + "[ log] h (upstream-2): -(1)", + "[ log] h (upstream-3): -(1)", + "[ log] i (upstream-self): -(3)", + "[ log] i (upstream-1-self): -(4)", + "[ log] i (upstream-2-self): -(6)", + "[ log] j (upstream-self): -(8)", + "[ log] j (upstream-1-self): -(9)", + "[ log] j (upstream-2-self): -(11)", + "[ log] b (complex): --(1)", + "[ log] b (upstream-1-self): --(1)", + "[ log] b (upstream-1-self-upstream): --(1)", + "[ log] b (upstream-2-self): --(1)", + "[ log] b (upstream-self): --(1)", + "[ log] c (upstream-1-self): --(1)", + "[ log] c (upstream-2): --(1)", + "[ log] c (upstream-3): --(1)", + "[ log] d (upstream-1-self): --(1)", + "[ log] d (upstream-2): --(1)", + "[ log] d (upstream-3): --(1)", + "[ log] e (upstream-1-self): --(1)", + "[ log] e (upstream-2): --(1)", + "[ log] f (complex): --(1)", + "[ log] f (upstream-1-self): --(1)", + "[ log] f (upstream-1-self-upstream): --(1)", + "[ log] f (upstream-2-self): --(1)", + "[ log] f (upstream-self): --(1)", + "[ log] g (complex): --(1)", + "[ log] g (upstream-1-self): --(1)", + "[ log] g (upstream-1-self-upstream): --(1)", + "[ log] g (upstream-2-self): --(1)", + "[ log] g (upstream-self): --(1)", + "[ log] h (complex): --(1)", + "[ log] h (upstream-1-self): --(1)", + "[ log] h (upstream-1-self-upstream): --(1)", + "[ log] h (upstream-2-self): --(1)", + "[ log] h (upstream-self): --(1)", + "[ log] c (complex): ---(1)", + "[ log] c (upstream-1-self-upstream): ---(1)", + "[ log] c (upstream-2-self): ---(1)", + "[ log] c (upstream-self): ---(1)", + "[ log] d (complex): ---(1)", + "[ log] d (upstream-1-self-upstream): ---(1)", + "[ log] d (upstream-2-self): ---(1)", + "[ log] d (upstream-self): ---(1)", + "[ log] e (upstream-1-self-upstream): ---(1)", + "[ log] e (upstream-2-self): ---(1)", + "[ log] e (upstream-3): ---(1)", + "[ log] e (complex): ----(1)", + "[ log] e (upstream-self): ----(1)", "[ log] ##################################################", "[ log] Cluster 0:", "[ log] - Dependencies: none", "[ log] - Clustered by: ", - "[ log] - (a (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (b (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (a (upstream-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (c (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (b (upstream-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (d (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (e (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (c (upstream-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (f (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (h (upstream-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (h (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (g (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (a (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: a (complex), a (upstream-3)", + "[ log] --------------------------------------------------", + "[ log] Cluster 1:", + "[ log] - Dependencies: none", + "[ log] - Clustered by: ", "[ log] - (a (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (b (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (c (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (h (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (a (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (b (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (c (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (h (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (d (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (e (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (f (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (g (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (d (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (e (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (f (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (g (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (a (upstream-1-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (b (upstream-1-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (c (upstream-1-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - (h (upstream-1-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (a (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (b (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (a (upstream-1-self-upstream)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (a (upstream-2-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (b (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (a (upstream-1-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (b (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (b (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (a (upstream-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (c (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (b (upstream-1-self-upstream)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (b (upstream-2-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (c (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (b (upstream-1-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (c (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (c (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (b (upstream-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (d (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (d (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (d (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (d (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (e (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (c (upstream-1-self-upstream)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (c (upstream-2-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (e (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (c (upstream-1-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (e (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (e (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (c (upstream-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (f (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (h (upstream-1-self-upstream)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (h (upstream-2-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (h (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (f (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (h (upstream-1-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (h (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (f (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (h (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (f (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (h (upstream-self)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (g (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (g (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (g (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - (g (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", "[ log] - (h (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: a (no-deps), b (no-deps), c (no-deps), d (no-deps), e (no-deps), f (no-deps), g (no-deps), h (no-deps), a (upstream-self), b (upstream-self), c (upstream-self), d (upstream-self), e (upstream-self), f (upstream-self), h (upstream-self), g (upstream-self), a (upstream-1), b (upstream-1), c (upstream-1), d (upstream-1), e (upstream-1), f (upstream-1), g (upstream-1), h (upstream-1), a (upstream-2), b (upstream-2), c (upstream-2), d (upstream-2), e (upstream-2), f (upstream-2), g (upstream-2), h (upstream-2), b (upstream-3), c (upstream-3), d (upstream-3), e (upstream-3), f (upstream-3), g (upstream-3), h (upstream-3), a (upstream-1-self), b (upstream-1-self), c (upstream-1-self), d (upstream-1-self), e (upstream-1-self), f (upstream-1-self), g (upstream-1-self), h (upstream-1-self), a (upstream-2-self), b (upstream-2-self), c (upstream-2-self), d (upstream-2-self), e (upstream-2-self), f (upstream-2-self), g (upstream-2-self), h (upstream-2-self), a (upstream-1-self-upstream), b (upstream-1-self-upstream), c (upstream-1-self-upstream), d (upstream-1-self-upstream), e (upstream-1-self-upstream), f (upstream-1-self-upstream), g (upstream-1-self-upstream), h (upstream-1-self-upstream), b (complex), c (complex), d (complex), e (complex), f (complex), g (complex), h (complex)", - "[ log] --------------------------------------------------", - "[ log] Cluster 1:", - "[ log] - Dependencies: none", - "[ log] - Clustered by: ", - "[ log] - (i (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: i (no-deps), i (upstream-self)", + "[ log] - Operations: a (no-deps), a (upstream-1), a (upstream-1-self), a (upstream-1-self-upstream), a (upstream-2), a (upstream-2-self), a (upstream-self), b (complex), b (no-deps), b (upstream-1), b (upstream-1-self), b (upstream-1-self-upstream), b (upstream-2), b (upstream-2-self), b (upstream-3), b (upstream-self), c (complex), c (no-deps), c (upstream-1), c (upstream-1-self), c (upstream-1-self-upstream), c (upstream-2), c (upstream-2-self), c (upstream-3), c (upstream-self), d (complex), d (no-deps), d (upstream-1), d (upstream-1-self), d (upstream-1-self-upstream), d (upstream-2), d (upstream-2-self), d (upstream-3), d (upstream-self), e (complex), e (no-deps), e (upstream-1), e (upstream-1-self), e (upstream-1-self-upstream), e (upstream-2), e (upstream-2-self), e (upstream-3), e (upstream-self), f (complex), f (no-deps), f (upstream-1), f (upstream-1-self), f (upstream-1-self-upstream), f (upstream-2), f (upstream-2-self), f (upstream-3), f (upstream-self), g (complex), g (no-deps), g (upstream-1), g (upstream-1-self), g (upstream-1-self-upstream), g (upstream-2), g (upstream-2-self), g (upstream-3), g (upstream-self), h (complex), h (no-deps), h (upstream-1), h (upstream-1-self), h (upstream-1-self-upstream), h (upstream-2), h (upstream-2-self), h (upstream-3), h (upstream-self)", "[ log] --------------------------------------------------", "[ log] Cluster 2:", "[ log] - Dependencies: none", "[ log] - Clustered by: ", - "[ log] - (j (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: j (no-deps), j (upstream-self)", + "[ log] - (i (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: i (complex), i (upstream-3)", "[ log] --------------------------------------------------", "[ log] Cluster 3:", "[ log] - Dependencies: none", "[ log] - Clustered by: ", - "[ log] - (i (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: i (upstream-1), i (upstream-1-self)", + "[ log] - (i (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: i (no-deps), i (upstream-self)", "[ log] --------------------------------------------------", "[ log] Cluster 4:", "[ log] - Dependencies: none", "[ log] - Clustered by: ", - "[ log] - (j (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: j (upstream-1), j (upstream-1-self)", + "[ log] - (i (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: i (upstream-1), i (upstream-1-self)", "[ log] --------------------------------------------------", "[ log] Cluster 5:", "[ log] - Dependencies: none", - "[ log] - Clustered by: ", - "[ log] - (i (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: i (upstream-2), i (upstream-2-self)", + "[ log] - Operations: i (upstream-1-self-upstream)", "[ log] --------------------------------------------------", "[ log] Cluster 6:", "[ log] - Dependencies: none", "[ log] - Clustered by: ", - "[ log] - (j (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: j (upstream-2), j (upstream-2-self)", + "[ log] - (i (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: i (upstream-2), i (upstream-2-self)", "[ log] --------------------------------------------------", "[ log] Cluster 7:", "[ log] - Dependencies: none", "[ log] - Clustered by: ", - "[ log] - (a (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: a (upstream-3), a (complex)", + "[ log] - (j (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: j (complex), j (upstream-3)", "[ log] --------------------------------------------------", "[ log] Cluster 8:", "[ log] - Dependencies: none", "[ log] - Clustered by: ", - "[ log] - (i (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: i (upstream-3), i (complex)", + "[ log] - (j (no-deps)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: j (no-deps), j (upstream-self)", "[ log] --------------------------------------------------", "[ log] Cluster 9:", "[ log] - Dependencies: none", "[ log] - Clustered by: ", - "[ log] - (j (upstream-3)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", - "[ log] - Operations: j (upstream-3), j (complex)", + "[ log] - (j (upstream-1)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: j (upstream-1), j (upstream-1-self)", "[ log] --------------------------------------------------", "[ log] Cluster 10:", "[ log] - Dependencies: none", - "[ log] - Operations: i (upstream-1-self-upstream)", + "[ log] - Operations: j (upstream-1-self-upstream)", "[ log] --------------------------------------------------", "[ log] Cluster 11:", "[ log] - Dependencies: none", - "[ log] - Operations: j (upstream-1-self-upstream)", + "[ log] - Clustered by: ", + "[ log] - (j (upstream-2)) \\"Project does not have a rush-project.json configuration file, or one provided by a rig, so it does not support caching.\\"", + "[ log] - Operations: j (upstream-2), j (upstream-2-self)", "[ log] --------------------------------------------------", "[ log] ##################################################", ] diff --git a/libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationExecutionManager.test.ts.snap b/libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationGraph.test.ts.snap similarity index 95% rename from libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationExecutionManager.test.ts.snap rename to libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationGraph.test.ts.snap index deb3e8fe623..c2956f4495f 100644 --- a/libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationExecutionManager.test.ts.snap +++ b/libraries/rush-lib/src/logic/operations/test/__snapshots__/OperationGraph.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`OperationExecutionManager Cobuild logging logs cobuilt operations correctly with --timeline option 1`] = ` +exports[`OperationGraph Cobuild logging logs cobuilt operations correctly with --timeline option 1`] = ` Array [ Object { "kind": "O", @@ -160,7 +160,7 @@ Array [ ] `; -exports[`OperationExecutionManager Cobuild logging logs warnings correctly with --timeline option 1`] = ` +exports[`OperationGraph Cobuild logging logs warnings correctly with --timeline option 1`] = ` Array [ Object { "kind": "O", @@ -352,7 +352,7 @@ Array [ ] `; -exports[`OperationExecutionManager Error logging printedStderrAfterError 1`] = ` +exports[`OperationGraph Error logging printedStderrAfterError 1`] = ` Array [ Object { "kind": "O", @@ -440,7 +440,7 @@ Array [ ] `; -exports[`OperationExecutionManager Error logging printedStdoutAfterErrorWithEmptyStderr 1`] = ` +exports[`OperationGraph Error logging printedStdoutAfterErrorWithEmptyStderr 1`] = ` Array [ Object { "kind": "O", @@ -528,7 +528,7 @@ Array [ ] `; -exports[`OperationExecutionManager Warning logging Fail on warning Logs warnings correctly 1`] = ` +exports[`OperationGraph Warning logging Fail on warning Logs warnings correctly 1`] = ` Array [ Object { "kind": "O", @@ -616,7 +616,7 @@ Array [ ] `; -exports[`OperationExecutionManager Warning logging Success on warning Logs warnings correctly 1`] = ` +exports[`OperationGraph Warning logging Success on warning Logs warnings correctly 1`] = ` Array [ Object { "kind": "O", @@ -698,7 +698,7 @@ Array [ ] `; -exports[`OperationExecutionManager Warning logging Success on warning logs warnings correctly with --timeline option 1`] = ` +exports[`OperationGraph Warning logging Success on warning logs warnings correctly with --timeline option 1`] = ` Array [ Object { "kind": "O", diff --git a/libraries/rush-lib/src/cli/parsing/test/__snapshots__/ParseParallelism.test.ts.snap b/libraries/rush-lib/src/logic/operations/test/__snapshots__/ParseParallelism.test.ts.snap similarity index 100% rename from libraries/rush-lib/src/cli/parsing/test/__snapshots__/ParseParallelism.test.ts.snap rename to libraries/rush-lib/src/logic/operations/test/__snapshots__/ParseParallelism.test.ts.snap diff --git a/libraries/rush-lib/src/logic/operations/test/__snapshots__/PhasedOperationPlugin.test.ts.snap b/libraries/rush-lib/src/logic/operations/test/__snapshots__/PhasedOperationPlugin.test.ts.snap index 35f3b43a828..16289e969da 100644 --- a/libraries/rush-lib/src/logic/operations/test/__snapshots__/PhasedOperationPlugin.test.ts.snap +++ b/libraries/rush-lib/src/logic/operations/test/__snapshots__/PhasedOperationPlugin.test.ts.snap @@ -95,29 +95,6 @@ Array [ ] `; -exports[`PhasedOperationPlugin handles different phaseOriginal vs phaseSelection cross-project with --include-phase-deps: multiple-project 1`] = ` -Array [ - "a (no-deps) (enabled) -> []", - "h (upstream-1) (enabled) -> [a (no-deps)]", - "a (upstream-1) (disabled) -> []", - "h (no-deps) (disabled) -> []", -] -`; - -exports[`PhasedOperationPlugin handles different phaseOriginal vs phaseSelection with --include-phase-deps: single-project 1`] = ` -Array [ - "a (no-deps) (enabled) -> []", - "a (upstream-self) (enabled) -> [a (no-deps)]", -] -`; - -exports[`PhasedOperationPlugin handles different phaseOriginal vs phaseSelection without --include-phase-deps: single-project 1`] = ` -Array [ - "a (no-deps) (enabled) -> []", - "a (upstream-self) (enabled) -> [a (no-deps)]", -] -`; - exports[`PhasedOperationPlugin handles filtered phases on filtered projects: missing-links 1`] = ` Array [ "a (complex) (enabled) -> [a (upstream-3)]", @@ -310,53 +287,28 @@ Array [ ] `; -exports[`PhasedOperationPlugin handles some changed projects within filtered projects: multiple 1`] = ` +exports[`PhasedOperationPlugin handles incomplete phaseSelection cross-project with --include-phase-deps: multiple-project 1`] = ` +Array [ + "a (no-deps) (enabled) -> []", + "h (upstream-1) (enabled) -> [a (no-deps)]", +] +`; + +exports[`PhasedOperationPlugin handles incomplete phaseSelection with --include-phase-deps: single-project 1`] = ` Array [ - "a (complex) (enabled) -> [a (upstream-3)]", "a (no-deps) (enabled) -> []", - "a (upstream-1) (enabled) -> []", - "a (upstream-1-self) (enabled) -> [a (upstream-1)]", - "a (upstream-1-self-upstream) (enabled) -> []", - "a (upstream-2) (enabled) -> []", - "a (upstream-2-self) (enabled) -> [a (upstream-2)]", - "a (upstream-3) (enabled) -> []", "a (upstream-self) (enabled) -> [a (no-deps)]", - "c (complex) (enabled) -> [b (upstream-1-self-upstream), b (upstream-2-self), c (upstream-3)]", - "c (no-deps) (enabled) -> []", - "c (upstream-1) (enabled) -> [b (no-deps)]", - "c (upstream-1-self) (enabled) -> [c (upstream-1)]", - "c (upstream-1-self-upstream) (enabled) -> [b (upstream-1-self)]", - "c (upstream-2) (enabled) -> [b (upstream-1)]", - "c (upstream-2-self) (enabled) -> [c (upstream-2)]", - "c (upstream-3) (enabled) -> [b (upstream-2)]", - "c (upstream-self) (enabled) -> [b (upstream-self), c (no-deps)]", - "f (complex) (enabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), f (upstream-3), h (upstream-1-self-upstream), h (upstream-2-self)]", - "f (upstream-1) (enabled) -> [a (no-deps), h (no-deps)]", - "f (upstream-1-self) (enabled) -> [f (upstream-1)]", - "f (upstream-1-self-upstream) (enabled) -> [a (upstream-1-self), h (upstream-1-self)]", - "f (upstream-2) (enabled) -> [a (upstream-1), h (upstream-1)]", - "f (upstream-2-self) (enabled) -> [f (upstream-2)]", - "f (upstream-3) (enabled) -> [a (upstream-2), h (upstream-2)]", - "f (upstream-self) (enabled) -> [a (upstream-self), f (no-deps), h (upstream-self)]", - "b (no-deps) (disabled) -> []", - "b (upstream-1) (disabled) -> [a (no-deps)]", - "b (upstream-1-self) (disabled) -> [b (upstream-1)]", - "b (upstream-1-self-upstream) (disabled) -> [a (upstream-1-self)]", - "b (upstream-2) (disabled) -> [a (upstream-1)]", - "b (upstream-2-self) (disabled) -> [b (upstream-2)]", - "b (upstream-self) (disabled) -> [a (upstream-self), b (no-deps)]", - "f (no-deps) (disabled) -> []", - "h (no-deps) (disabled) -> []", - "h (upstream-1) (disabled) -> [a (no-deps)]", - "h (upstream-1-self) (disabled) -> [h (upstream-1)]", - "h (upstream-1-self-upstream) (disabled) -> [a (upstream-1-self)]", - "h (upstream-2) (disabled) -> [a (upstream-1)]", - "h (upstream-2-self) (disabled) -> [h (upstream-2)]", - "h (upstream-self) (disabled) -> [a (upstream-self), h (no-deps)]", ] `; -exports[`PhasedOperationPlugin handles some changed projects: multiple 1`] = ` +exports[`PhasedOperationPlugin handles incomplete phaseSelection without --include-phase-deps: single-project 1`] = ` +Array [ + "a (upstream-self) (enabled) -> [a (no-deps)]", + "a (no-deps) (disabled) -> []", +] +`; + +exports[`PhasedOperationPlugin includes full graph but enables subset when generateFullGraph is true: full-graph-filtered 1`] = ` Array [ "a (complex) (enabled) -> [a (upstream-3)]", "a (no-deps) (enabled) -> []", @@ -367,14 +319,6 @@ Array [ "a (upstream-2-self) (enabled) -> [a (upstream-2)]", "a (upstream-3) (enabled) -> []", "a (upstream-self) (enabled) -> [a (no-deps)]", - "b (complex) (enabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), b (upstream-3)]", - "b (upstream-1) (enabled) -> [a (no-deps)]", - "b (upstream-1-self) (enabled) -> [b (upstream-1)]", - "b (upstream-1-self-upstream) (enabled) -> [a (upstream-1-self)]", - "b (upstream-2) (enabled) -> [a (upstream-1)]", - "b (upstream-2-self) (enabled) -> [b (upstream-2)]", - "b (upstream-3) (enabled) -> [a (upstream-2)]", - "b (upstream-self) (enabled) -> [a (upstream-self), b (no-deps)]", "c (complex) (enabled) -> [b (upstream-1-self-upstream), b (upstream-2-self), c (upstream-3)]", "c (no-deps) (enabled) -> []", "c (upstream-1) (enabled) -> [b (no-deps)]", @@ -384,93 +328,6 @@ Array [ "c (upstream-2-self) (enabled) -> [c (upstream-2)]", "c (upstream-3) (enabled) -> [b (upstream-2)]", "c (upstream-self) (enabled) -> [b (upstream-self), c (no-deps)]", - "d (complex) (enabled) -> [b (upstream-1-self-upstream), b (upstream-2-self), d (upstream-3)]", - "d (upstream-1-self-upstream) (enabled) -> [b (upstream-1-self)]", - "d (upstream-2) (enabled) -> [b (upstream-1)]", - "d (upstream-2-self) (enabled) -> [d (upstream-2)]", - "d (upstream-3) (enabled) -> [b (upstream-2)]", - "d (upstream-self) (enabled) -> [b (upstream-self), d (no-deps)]", - "e (complex) (enabled) -> [c (upstream-1-self-upstream), c (upstream-2-self), e (upstream-3)]", - "e (upstream-1) (enabled) -> [c (no-deps)]", - "e (upstream-1-self) (enabled) -> [e (upstream-1)]", - "e (upstream-1-self-upstream) (enabled) -> [c (upstream-1-self)]", - "e (upstream-2) (enabled) -> [c (upstream-1)]", - "e (upstream-2-self) (enabled) -> [e (upstream-2)]", - "e (upstream-3) (enabled) -> [c (upstream-2)]", - "e (upstream-self) (enabled) -> [c (upstream-self), e (no-deps)]", - "f (complex) (enabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), f (upstream-3), h (upstream-1-self-upstream), h (upstream-2-self)]", - "f (no-deps) (enabled) -> []", - "f (upstream-1) (enabled) -> [a (no-deps), h (no-deps)]", - "f (upstream-1-self) (enabled) -> [f (upstream-1)]", - "f (upstream-1-self-upstream) (enabled) -> [a (upstream-1-self), h (upstream-1-self)]", - "f (upstream-2) (enabled) -> [a (upstream-1), h (upstream-1)]", - "f (upstream-2-self) (enabled) -> [f (upstream-2)]", - "f (upstream-3) (enabled) -> [a (upstream-2), h (upstream-2)]", - "f (upstream-self) (enabled) -> [a (upstream-self), f (no-deps), h (upstream-self)]", - "g (complex) (enabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), g (upstream-3)]", - "g (upstream-1) (enabled) -> [a (no-deps)]", - "g (upstream-1-self) (enabled) -> [g (upstream-1)]", - "g (upstream-1-self-upstream) (enabled) -> [a (upstream-1-self)]", - "g (upstream-2) (enabled) -> [a (upstream-1)]", - "g (upstream-2-self) (enabled) -> [g (upstream-2)]", - "g (upstream-3) (enabled) -> [a (upstream-2)]", - "g (upstream-self) (enabled) -> [a (upstream-self), g (no-deps)]", - "h (complex) (enabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), h (upstream-3)]", - "h (upstream-1) (enabled) -> [a (no-deps)]", - "h (upstream-1-self) (enabled) -> [h (upstream-1)]", - "h (upstream-1-self-upstream) (enabled) -> [a (upstream-1-self)]", - "h (upstream-2) (enabled) -> [a (upstream-1)]", - "h (upstream-2-self) (enabled) -> [h (upstream-2)]", - "h (upstream-3) (enabled) -> [a (upstream-2)]", - "h (upstream-self) (enabled) -> [a (upstream-self), h (no-deps)]", - "b (no-deps) (disabled) -> []", - "d (no-deps) (disabled) -> []", - "d (upstream-1) (disabled) -> [b (no-deps)]", - "d (upstream-1-self) (disabled) -> [d (upstream-1)]", - "e (no-deps) (disabled) -> []", - "g (no-deps) (disabled) -> []", - "h (no-deps) (disabled) -> []", - "i (complex) (disabled) -> [i (upstream-3)]", - "i (no-deps) (disabled) -> []", - "i (upstream-1) (disabled) -> []", - "i (upstream-1-self) (disabled) -> [i (upstream-1)]", - "i (upstream-1-self-upstream) (disabled) -> []", - "i (upstream-2) (disabled) -> []", - "i (upstream-2-self) (disabled) -> [i (upstream-2)]", - "i (upstream-3) (disabled) -> []", - "i (upstream-self) (disabled) -> [i (no-deps)]", - "j (complex) (disabled) -> [j (upstream-3)]", - "j (no-deps) (disabled) -> []", - "j (upstream-1) (disabled) -> []", - "j (upstream-1-self) (disabled) -> [j (upstream-1)]", - "j (upstream-1-self-upstream) (disabled) -> []", - "j (upstream-2) (disabled) -> []", - "j (upstream-2-self) (disabled) -> [j (upstream-2)]", - "j (upstream-3) (disabled) -> []", - "j (upstream-self) (disabled) -> [j (no-deps)]", -] -`; - -exports[`PhasedOperationPlugin handles some changed projects: single 1`] = ` -Array [ - "g (complex) (enabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), g (upstream-3)]", - "g (no-deps) (enabled) -> []", - "g (upstream-1) (enabled) -> [a (no-deps)]", - "g (upstream-1-self) (enabled) -> [g (upstream-1)]", - "g (upstream-1-self-upstream) (enabled) -> [a (upstream-1-self)]", - "g (upstream-2) (enabled) -> [a (upstream-1)]", - "g (upstream-2-self) (enabled) -> [g (upstream-2)]", - "g (upstream-3) (enabled) -> [a (upstream-2)]", - "g (upstream-self) (enabled) -> [a (upstream-self), g (no-deps)]", - "a (complex) (disabled) -> [a (upstream-3)]", - "a (no-deps) (disabled) -> []", - "a (upstream-1) (disabled) -> []", - "a (upstream-1-self) (disabled) -> [a (upstream-1)]", - "a (upstream-1-self-upstream) (disabled) -> []", - "a (upstream-2) (disabled) -> []", - "a (upstream-2-self) (disabled) -> [a (upstream-2)]", - "a (upstream-3) (disabled) -> []", - "a (upstream-self) (disabled) -> [a (no-deps)]", "b (complex) (disabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), b (upstream-3)]", "b (no-deps) (disabled) -> []", "b (upstream-1) (disabled) -> [a (no-deps)]", @@ -480,15 +337,6 @@ Array [ "b (upstream-2-self) (disabled) -> [b (upstream-2)]", "b (upstream-3) (disabled) -> [a (upstream-2)]", "b (upstream-self) (disabled) -> [a (upstream-self), b (no-deps)]", - "c (complex) (disabled) -> [b (upstream-1-self-upstream), b (upstream-2-self), c (upstream-3)]", - "c (no-deps) (disabled) -> []", - "c (upstream-1) (disabled) -> [b (no-deps)]", - "c (upstream-1-self) (disabled) -> [c (upstream-1)]", - "c (upstream-1-self-upstream) (disabled) -> [b (upstream-1-self)]", - "c (upstream-2) (disabled) -> [b (upstream-1)]", - "c (upstream-2-self) (disabled) -> [c (upstream-2)]", - "c (upstream-3) (disabled) -> [b (upstream-2)]", - "c (upstream-self) (disabled) -> [b (upstream-self), c (no-deps)]", "d (complex) (disabled) -> [b (upstream-1-self-upstream), b (upstream-2-self), d (upstream-3)]", "d (no-deps) (disabled) -> []", "d (upstream-1) (disabled) -> [b (no-deps)]", @@ -516,6 +364,15 @@ Array [ "f (upstream-2-self) (disabled) -> [f (upstream-2)]", "f (upstream-3) (disabled) -> [a (upstream-2), h (upstream-2)]", "f (upstream-self) (disabled) -> [a (upstream-self), f (no-deps), h (upstream-self)]", + "g (complex) (disabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), g (upstream-3)]", + "g (no-deps) (disabled) -> []", + "g (upstream-1) (disabled) -> [a (no-deps)]", + "g (upstream-1-self) (disabled) -> [g (upstream-1)]", + "g (upstream-1-self-upstream) (disabled) -> [a (upstream-1-self)]", + "g (upstream-2) (disabled) -> [a (upstream-1)]", + "g (upstream-2-self) (disabled) -> [g (upstream-2)]", + "g (upstream-3) (disabled) -> [a (upstream-2)]", + "g (upstream-self) (disabled) -> [a (upstream-self), g (no-deps)]", "h (complex) (disabled) -> [a (upstream-1-self-upstream), a (upstream-2-self), h (upstream-3)]", "h (no-deps) (disabled) -> []", "h (upstream-1) (disabled) -> [a (no-deps)]", diff --git a/libraries/rush-lib/src/pluginFramework/OperationGraphHooks.ts b/libraries/rush-lib/src/pluginFramework/OperationGraphHooks.ts new file mode 100644 index 00000000000..e04196f60ce --- /dev/null +++ b/libraries/rush-lib/src/pluginFramework/OperationGraphHooks.ts @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + AsyncSeriesBailHook, + AsyncSeriesHook, + AsyncSeriesWaterfallHook, + SyncHook, + SyncWaterfallHook +} from 'tapable'; + +import type { Operation } from '../logic/operations/Operation'; +import type { + IOperationExecutionResult, + IConfigurableOperation +} from '../logic/operations/IOperationExecutionResult'; +import type { OperationStatus } from '../logic/operations/OperationStatus'; +import type { IOperationRunnerContext } from '../logic/operations/IOperationRunner'; +import type { ITelemetryData } from '../logic/Telemetry'; +import type { IEnvironment } from '../utilities/Utilities'; +import type { IOperationGraph, IOperationGraphIterationOptions } from '../logic/operations/IOperationGraph'; + +/** + * Hooks into the execution process for operations within the graph. + * + * Per-iteration lifecycle: + * 1. `configureIteration` - Synchronously decide which operations to enable for the next iteration. + * 2. `onIterationScheduled` - Fires after the iteration is prepared but before execution begins, if it has any enabled operations. + * 3. `beforeExecuteIterationAsync` - Async hook that can bail out the iteration entirely. + * 4. Operations execute (status changes reported via `onExecutionStatesUpdated`). + * 5. `afterExecuteIterationAsync` - Fires after all operations in the iteration have settled. + * 6. `onIdle` - Fires when the graph enters idle state awaiting changes (watch mode only). + * + * Additional hooks: + * - `onEnableStatesChanged` - Fires when `setEnabledStates` mutates operation enabled flags. + * - `onInvalidateOperations` - Fires when operations are invalidated (e.g. by file watchers). + * - `onGraphStateChanged` - Fires on any observable graph state change. + * + * @alpha + */ +export class OperationGraphHooks { + /** + * Hook invoked to decide what work a potential new iteration contains. + * Use the `lastExecutedRecords` to determine which operations are new or have had their inputs changed. + * Set the `enabled` states on the values in `initialRecords` to control which operations will be executed. + * + * @remarks + * This hook is synchronous to guarantee that the `lastExecutedRecords` map remains stable for the + * duration of configuration. This hook often executes while an execution iteration is currently running, so + * operations could complete if there were async ticks during the configuration phase. + * + * If no operations are marked for execution, the iteration will not be scheduled. + * If there is an existing scheduled iteration, it will remain. + */ + public readonly configureIteration: SyncHook< + [ + ReadonlyMap, + ReadonlyMap, + IOperationGraphIterationOptions + ] + > = new SyncHook(['initialRecords', 'lastExecutedRecords', 'context'], 'configureIteration'); + + /** + * Hook invoked before operation start for an iteration. Allows a plugin to perform side-effects or + * short-circuit the entire iteration. + * + * If any tap returns an {@link OperationStatus}, the remaining taps are skipped and the iteration will + * end immediately with that status. All operations which have not yet executed will be marked + * Aborted. + */ + public readonly beforeExecuteIterationAsync: AsyncSeriesBailHook< + [ReadonlyMap, IOperationGraphIterationOptions], + OperationStatus | undefined | void + > = new AsyncSeriesBailHook(['records', 'context'], 'beforeExecuteIterationAsync'); + + /** + * Batched hook invoked when one or more operation statuses have changed during the same microtask. + * The hook receives an array of the operation execution results that changed status. + * @remarks + * This hook is batched to reduce noise when updating many operations synchronously in quick succession. + */ + public readonly onExecutionStatesUpdated: SyncHook<[ReadonlySet]> = new SyncHook( + ['records'], + 'onExecutionStatesUpdated' + ); + + /** + * Hook invoked when one or more operations have their enabled state mutated via + * {@link IOperationGraph.setEnabledStates}. Provides the set of operations whose + * enabled state actually changed. + */ + public readonly onEnableStatesChanged: SyncHook<[ReadonlySet]> = new SyncHook( + ['operations'], + 'onEnableStatesChanged' + ); + + /** + * Hook invoked immediately after a new execution iteration is scheduled (i.e. operations selected and prepared), + * before any operations in that iteration have started executing. Can be used to snapshot planned work, + * drive UIs, or pre-compute auxiliary data. + */ + public readonly onIterationScheduled: SyncHook<[ReadonlyMap]> = + new SyncHook(['records'], 'onIterationScheduled'); + + /** + * Hook invoked when any observable state on the operation graph changes. + * This includes configuration mutations (parallelism, quiet/debug modes, pauseNextIteration) + * as well as dynamic state (status transitions, scheduled iteration availability, etc.). + * Hook is series for stable output. + */ + public readonly onGraphStateChanged: SyncHook<[IOperationGraph]> = new SyncHook( + ['operationGraph'], + 'onGraphStateChanged' + ); + + /** + * Hook invoked when operations are invalidated for any reason. + */ + public readonly onInvalidateOperations: SyncHook<[Iterable, string | undefined]> = new SyncHook( + ['operations', 'reason'], + 'onInvalidateOperations' + ); + + /** + * Hook invoked after an iteration has finished and the command is watching for changes. + * May be used to display additional relevant data to the user. + * Only relevant when running in watch mode. + */ + public readonly onIdle: SyncHook = new SyncHook(undefined, 'onIdle'); + + /** + * Hook invoked after executing a set of operations. + * Hook is series for stable output. + */ + public readonly afterExecuteIterationAsync: AsyncSeriesWaterfallHook< + [OperationStatus, ReadonlyMap, IOperationGraphIterationOptions] + > = new AsyncSeriesWaterfallHook(['status', 'results', 'context'], 'afterExecuteIterationAsync'); + + /** + * Hook invoked after executing an iteration, before the telemetry entry is written. + * Allows the caller to augment or modify the log entry. + */ + public readonly beforeLog: SyncHook = new SyncHook(['telemetryData'], 'beforeLog'); + + /** + * Hook invoked before executing a operation. + */ + public readonly beforeExecuteOperationAsync: AsyncSeriesBailHook< + [IOperationRunnerContext & IOperationExecutionResult], + OperationStatus | undefined + > = new AsyncSeriesBailHook(['runnerContext'], 'beforeExecuteOperationAsync'); + + /** + * Hook invoked to define environment variables for an operation. + * May be invoked by the runner to get the environment for the operation. + */ + public readonly createEnvironmentForOperation: SyncWaterfallHook< + [IEnvironment, IOperationRunnerContext & IOperationExecutionResult] + > = new SyncWaterfallHook(['environment', 'runnerContext'], 'createEnvironmentForOperation'); + + /** + * Hook invoked after executing a operation. + */ + public readonly afterExecuteOperationAsync: AsyncSeriesHook< + [IOperationRunnerContext & IOperationExecutionResult] + > = new AsyncSeriesHook(['runnerContext'], 'afterExecuteOperationAsync'); +} diff --git a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts index 97ccdb064aa..d3a7fc15074 100644 --- a/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts +++ b/libraries/rush-lib/src/pluginFramework/PhasedCommandHooks.ts @@ -1,14 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import { - AsyncParallelHook, - AsyncSeriesBailHook, - AsyncSeriesHook, - AsyncSeriesWaterfallHook, - SyncHook, - SyncWaterfallHook -} from 'tapable'; +import { AsyncSeriesHook, AsyncSeriesWaterfallHook } from 'tapable'; import type { CommandLineParameter } from '@rushstack/ts-command-line'; @@ -17,17 +10,11 @@ import type { IPhase } from '../api/CommandLineConfiguration'; import type { RushConfiguration } from '../api/RushConfiguration'; import type { RushConfigurationProject } from '../api/RushConfigurationProject'; import type { Operation } from '../logic/operations/Operation'; -import type { - IExecutionResult, - IOperationExecutionResult -} from '../logic/operations/IOperationExecutionResult'; import type { CobuildConfiguration } from '../api/CobuildConfiguration'; import type { RushProjectConfiguration } from '../api/RushProjectConfiguration'; -import type { IOperationRunnerContext } from '../logic/operations/IOperationRunner'; -import type { ITelemetryData } from '../logic/Telemetry'; -import type { OperationStatus } from '../logic/operations/OperationStatus'; +import type { Parallelism } from '../logic/operations/ParseParallelism'; import type { IInputsSnapshot } from '../logic/incremental/InputsSnapshot'; -import type { IEnvironment } from '../utilities/Utilities'; +import type { IOperationGraph } from '../logic/operations/IOperationGraph'; /** * A plugin that interacts with a phased commands. @@ -64,16 +51,15 @@ export interface ICreateOperationsContext { * Maps from the `longName` field in command-line.json to the parser configuration in ts-command-line. */ readonly customParameters: ReadonlyMap; + /** + * If true, dependencies of the selected phases will be automatically enabled in the execution. + */ + readonly includePhaseDeps: boolean; /** * If true, projects may read their output from cache or be skipped if already up to date. * If false, neither of the above may occur, e.g. "rush rebuild" */ readonly isIncrementalBuildAllowed: boolean; - /** - * If true, this is the initial run of the command. - * If false, this execution is in response to changes. - */ - readonly isInitial: boolean; /** * If true, the command is running in watch mode. */ @@ -81,136 +67,64 @@ export interface ICreateOperationsContext { /** * The currently configured maximum parallelism for the command. */ - readonly parallelism: number; - /** - * The set of phases original for the current command execution. - */ - readonly phaseOriginal: ReadonlySet; + readonly parallelism: Parallelism; /** - * The set of phases selected for the current command execution. + * The set of phases selected for execution. */ readonly phaseSelection: ReadonlySet; - /** - * The set of Rush projects selected for the current command execution. - */ - readonly projectSelection: ReadonlySet; /** * All successfully loaded rush-project.json data for selected projects. */ readonly projectConfigurations: ReadonlyMap; /** - * The set of Rush projects that have not been built in the current process since they were last modified. - * When `isInitial` is true, this will be an exact match of `projectSelection`. + * The set of Rush projects selected for execution. */ - readonly projectsInUnknownState: ReadonlySet; - /** - * The Rush configuration - */ - readonly rushConfiguration: RushConfiguration; + readonly projectSelection: ReadonlySet; /** - * If true, Rush will automatically include the dependent phases for the specified set of phases. - * @remarks - * If the selection of projects was "unsafe" (i.e. missing some dependencies), this will add the - * minimum number of phases required to make it safe. + * If true, the operation graph should include all projects in the repository (watch broad graph mode). + * Only the projects in projectSelection should start enabled; others are present but disabled. */ - readonly includePhaseDeps: boolean; + readonly generateFullGraph?: boolean; /** - * Marks an operation's result as invalid, potentially triggering a new build. Only applicable in watch mode. - * @param operation - The operation to invalidate - * @param reason - The reason for invalidating the operation + * The Rush configuration */ - readonly invalidateOperation?: ((operation: Operation, reason: string) => void) | undefined; + readonly rushConfiguration: RushConfiguration; } /** - * Context used for executing operations. + * Context used for configuring the operation graph. * @alpha */ -export interface IExecuteOperationsContext extends ICreateOperationsContext { +export interface IOperationGraphContext extends ICreateOperationsContext { /** * The current state of the repository, if available. * Not part of the creation context to avoid the overhead of Git calls when initializing the graph. */ - readonly inputsSnapshot?: IInputsSnapshot; - - /** - * An abort controller that can be used to abort the current set of queued operations. - */ - readonly abortController: AbortController; + readonly initialSnapshot?: IInputsSnapshot; } /** - * Hooks into the execution process for phased commands + * Hooks into the execution process for phased commands. + * + * Lifecycle: + * 1. `createOperationsAsync` - Invoked to populate the set of operations for execution. + * 2. `onGraphCreatedAsync` - Invoked after the operation graph is created, allowing plugins to + * tap into graph-level hooks (e.g. `configureIteration`, `onIdle`). + * See {@link OperationGraphHooks} for the per-iteration lifecycle. + * * @alpha */ export class PhasedCommandHooks { /** * Hook invoked to create operations for execution. - * Use the context to distinguish between the initial run and phased runs. - */ - public readonly createOperations: AsyncSeriesWaterfallHook<[Set, ICreateOperationsContext]> = - new AsyncSeriesWaterfallHook(['operations', 'context'], 'createOperations'); - - /** - * Hook invoked before operation start - * Hook is series for stable output. - */ - public readonly beforeExecuteOperations: AsyncSeriesHook< - [Map, IExecuteOperationsContext] - > = new AsyncSeriesHook(['records', 'context']); - - /** - * Hook invoked when operation status changed - * Hook is series for stable output. - */ - public readonly onOperationStatusChanged: SyncHook<[IOperationExecutionResult]> = new SyncHook(['record']); - - /** - * Hook invoked after executing a set of operations. - * Use the context to distinguish between the initial run and phased runs. - * Hook is series for stable output. - */ - public readonly afterExecuteOperations: AsyncSeriesHook<[IExecutionResult, IExecuteOperationsContext]> = - new AsyncSeriesHook(['results', 'context']); - - /** - * Hook invoked before executing a operation. - */ - public readonly beforeExecuteOperation: AsyncSeriesBailHook< - [IOperationRunnerContext & IOperationExecutionResult], - OperationStatus | undefined - > = new AsyncSeriesBailHook(['runnerContext'], 'beforeExecuteOperation'); - - /** - * Hook invoked to define environment variables for an operation. - * May be invoked by the runner to get the environment for the operation. - */ - public readonly createEnvironmentForOperation: SyncWaterfallHook< - [IEnvironment, IOperationRunnerContext & IOperationExecutionResult] - > = new SyncWaterfallHook(['environment', 'runnerContext'], 'createEnvironmentForOperation'); - - /** - * Hook invoked after executing a operation. - */ - public readonly afterExecuteOperation: AsyncSeriesHook< - [IOperationRunnerContext & IOperationExecutionResult] - > = new AsyncSeriesHook(['runnerContext'], 'afterExecuteOperation'); - - /** - * Hook invoked to shutdown long-lived work in plugins. - */ - public readonly shutdownAsync: AsyncParallelHook = new AsyncParallelHook(undefined, 'shutdown'); - - /** - * Hook invoked after a run has finished and the command is watching for changes. - * May be used to display additional relevant data to the user. - * Only relevant when running in watch mode. */ - public readonly waitingForChanges: SyncHook = new SyncHook(undefined, 'waitingForChanges'); + public readonly createOperationsAsync: AsyncSeriesWaterfallHook< + [Set, ICreateOperationsContext] + > = new AsyncSeriesWaterfallHook(['operations', 'context'], 'createOperationsAsync'); /** - * Hook invoked after executing operations and before waitingForChanges. Allows the caller - * to augment or modify the log entry about to be written. + * Hook invoked when the operation graph is created, allowing the plugin to tap into it and interact with it. */ - public readonly beforeLog: SyncHook = new SyncHook(['telemetryData'], 'beforeLog'); + public readonly onGraphCreatedAsync: AsyncSeriesHook<[IOperationGraph, IOperationGraphContext]> = + new AsyncSeriesHook(['operationGraph', 'context'], 'onGraphCreatedAsync'); } diff --git a/libraries/rush-lib/src/schemas/command-line.schema.json b/libraries/rush-lib/src/schemas/command-line.schema.json index 0091e7bb7ae..4baa326e8f5 100644 --- a/libraries/rush-lib/src/schemas/command-line.schema.json +++ b/libraries/rush-lib/src/schemas/command-line.schema.json @@ -234,6 +234,11 @@ "items": { "type": "string" } + }, + "includeAllProjectsInWatchGraph": { + "title": "Include All Projects In Watch Graph", + "description": "If true, when entering watch mode Rush will construct the operation graph including every project in the repository (respecting phase selection), but will initially enable only those operations whose projects were selected by the user's CLI project selection parameters. Other projects will appear disabled until they change or become required by an enabled project's dependency graph. This can improve iteration by avoiding a full graph rebuild when broadening the selection mid-session.", + "type": "boolean" } } }, diff --git a/libraries/rush-lib/src/utilities/Stopwatch.ts b/libraries/rush-lib/src/utilities/Stopwatch.ts index b4e55df8e44..e137f27e140 100644 --- a/libraries/rush-lib/src/utilities/Stopwatch.ts +++ b/libraries/rush-lib/src/utilities/Stopwatch.ts @@ -63,8 +63,8 @@ export class Stopwatch implements IStopwatchResult { /** * Static helper function which creates a stopwatch which is immediately started */ - public static start(): Stopwatch { - return new Stopwatch().start(); + public static start(startTimeOverride?: number): Stopwatch { + return new Stopwatch().start(startTimeOverride); } public get state(): StopwatchState { @@ -75,11 +75,11 @@ export class Stopwatch implements IStopwatchResult { * Starts the stopwatch. Note that if end() has been called, * reset() should be called before calling start() again. */ - public start(): Stopwatch { + public start(startTimeOverride?: number): Stopwatch { if (this._startTime !== undefined) { throw new Error('Call reset() before starting the Stopwatch'); } - this._startTime = this._getTime(); + this._startTime = startTimeOverride ?? this._getTime(); this._endTime = undefined; this._state = StopwatchState.Started; return this; diff --git a/libraries/rush-lib/src/utilities/test/Stopwatch.test.ts b/libraries/rush-lib/src/utilities/test/Stopwatch.test.ts index 931707e2385..f15e9fcfd5f 100644 --- a/libraries/rush-lib/src/utilities/test/Stopwatch.test.ts +++ b/libraries/rush-lib/src/utilities/test/Stopwatch.test.ts @@ -111,4 +111,29 @@ describe(Stopwatch.name, () => { expect(watch.duration).toEqual(1); expect(watch.duration).toEqual(2); }); + + it('uses startTimeOverride when provided to start()', () => { + const watch: Stopwatch = new Stopwatch(pseudoTimeMilliseconds([5000])); + watch.start(2000); + expect(watch.startTime).toEqual(2000); + watch.stop(); + expect(watch.duration).toEqual(3); + }); + + it('uses startTimeOverride with the static start() shorthand', () => { + const watch: Stopwatch = Stopwatch.start(1000); + expect(watch.startTime).toEqual(1000); + expect(watch.state).toEqual(StopwatchState.Started); + }); + + it('ignores getTime for start when startTimeOverride is provided', () => { + const getTime = pseudoTimeMilliseconds([10000]); + const watch: Stopwatch = new Stopwatch(getTime); + watch.start(3000); + // startTime should be the override, not a value from getTime + expect(watch.startTime).toEqual(3000); + // stop still uses getTime + watch.stop(); + expect(watch.duration).toEqual(7); + }); }); diff --git a/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap b/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap index b497003e629..59020c926b3 100644 --- a/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap +++ b/libraries/rush-sdk/src/test/__snapshots__/script.test.ts.snap @@ -31,6 +31,7 @@ Loaded @microsoft/rush-lib from process.env._RUSH_LIB_PATH 'LookupByPath', 'NpmOptionsConfiguration', 'Operation', + 'OperationGraphHooks', 'OperationStatus', 'PackageJsonDependency', 'PackageJsonDependencyMeta', diff --git a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts index f0cd91e4d52..8a90dec57d8 100644 --- a/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts +++ b/rush-plugins/rush-bridge-cache-plugin/src/BridgeCachePlugin.ts @@ -2,19 +2,19 @@ // See LICENSE in the project root for license information. import { Async, FileSystem } from '@rushstack/node-core-library'; -import { _OperationBuildCache as OperationBuildCache } from '@rushstack/rush-sdk'; -import type { - ICreateOperationsContext, - IExecuteOperationsContext, - ILogger, - IOperationExecutionResult, - IPhasedCommand, - IRushPlugin, - Operation, - RushSession +import { + _OperationBuildCache as OperationBuildCache, + OperationStatus, + type Operation, + type IOperationExecutionResult, + type IOperationGraphIterationOptions, + type ILogger, + type IBaseOperationExecutionResult, + type IPhasedCommand, + type IRushPlugin, + type RushSession } from '@rushstack/rush-sdk'; -import { CommandLineParameterKind } from '@rushstack/ts-command-line'; -import type { CommandLineParameter } from '@rushstack/ts-command-line'; +import { CommandLineParameterKind, type CommandLineParameter } from '@rushstack/ts-command-line'; const PLUGIN_NAME: 'RushBridgeCachePlugin' = 'RushBridgeCachePlugin'; @@ -46,144 +46,122 @@ export class BridgeCachePlugin implements IRushPlugin { public apply(session: RushSession): void { session.hooks.runAnyPhasedCommand.tapPromise(PLUGIN_NAME, async (command: IPhasedCommand) => { - const logger: ILogger = session.getLogger(PLUGIN_NAME); - - let cacheAction: CacheAction | undefined; - let requireOutputFolders: boolean = false; - - // cancel the actual operations. We don't want to run the command, just cache the output folders on disk - command.hooks.createOperations.tap( - { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER }, - (operations: Set, context: ICreateOperationsContext): Set => { - const { customParameters } = context; - cacheAction = this._getCacheAction(customParameters); - - if (cacheAction !== undefined) { - if (!context.buildCacheConfiguration?.buildCacheEnabled) { - throw new Error( - `The build cache must be enabled to use the "${this._actionParameterName}" parameter.` - ); - } - - for (const operation of operations) { - operation.enabled = false; - } + command.hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, async (graph, context) => { + const { customParameters, buildCacheConfiguration } = context; + const cacheAction: CacheAction | undefined = this._getCacheAction(customParameters); - requireOutputFolders = this._isRequireOutputFoldersFlagSet(customParameters); + if (cacheAction !== undefined) { + if (!buildCacheConfiguration?.buildCacheEnabled) { + throw new Error( + `The build cache must be enabled to use the "${this._actionParameterName}" parameter.` + ); } - return operations; - } - ); - // populate the cache for each operation - command.hooks.beforeExecuteOperations.tapPromise( - PLUGIN_NAME, - async ( - recordByOperation: Map, - context: IExecuteOperationsContext - ): Promise => { const { - buildCacheConfiguration, rushConfiguration: { experimentsConfiguration: { configuration: { omitAppleDoubleFilesFromBuildCache } } } } = context; - const { terminal } = logger; - - if (cacheAction === undefined) { - return; - } - - if (!buildCacheConfiguration?.buildCacheEnabled) { - throw new Error( - `The build cache must be enabled to use the "${this._actionParameterName}" parameter.` - ); - } - - const filteredOperations: Set = new Set(); - for (const operationExecutionResult of recordByOperation.values()) { - if (!operationExecutionResult.operation.isNoOp) { - filteredOperations.add(operationExecutionResult); - } - } - let successCount: number = 0; - - await Async.forEachAsync( - filteredOperations, - async (operationExecutionResult: IOperationExecutionResult) => { - const projectBuildCache: OperationBuildCache = OperationBuildCache.forOperation( - operationExecutionResult, - { - buildCacheConfiguration, - terminal, - excludeAppleDoubleFiles: !!omitAppleDoubleFilesFromBuildCache + const logger: ILogger = session.getLogger(PLUGIN_NAME); + const { terminal } = logger; + const requireOutputFolders: boolean = this._isRequireOutputFoldersFlagSet(customParameters); + + graph.hooks.beforeExecuteIterationAsync.tapPromise( + PLUGIN_NAME, + async ( + operationRecords: ReadonlyMap, + iterationOptions: IOperationGraphIterationOptions + ): Promise => { + const filteredOperations: IBaseOperationExecutionResult[] = []; + for (const record of operationRecords.values()) { + if (!record.operation.isNoOp) { + filteredOperations.push(record); } - ); + } - const { operation } = operationExecutionResult; + if (!filteredOperations.length) { + return; // nothing to do, continue normal execution + } - if (cacheAction === CACHE_ACTION_READ) { - const success: boolean = await projectBuildCache.tryRestoreFromCacheAsync(terminal); - if (success) { - ++successCount; - terminal.writeLine( - `Operation "${operation.name}": Outputs have been restored from the build cache."` - ); - terminal.writeLine(`Cache key: ${projectBuildCache.cacheId}`); - } else { - terminal.writeWarningLine( - `Operation "${operation.name}": Outputs could not be restored from the build cache.` + let successCount: number = 0; + await Async.forEachAsync( + filteredOperations, + async (operationExecutionResult: IBaseOperationExecutionResult) => { + const projectBuildCache: OperationBuildCache = OperationBuildCache.forOperation( + operationExecutionResult, + { + buildCacheConfiguration, + terminal, + excludeAppleDoubleFiles: !!omitAppleDoubleFilesFromBuildCache + } ); - } - } else if (cacheAction === CACHE_ACTION_WRITE) { - // if the require output folders flag has been passed, skip populating the cache if any of the expected output folders does not exist - if ( - requireOutputFolders && - operation.settings?.outputFolderNames && - operation.settings?.outputFolderNames?.length > 0 - ) { - const projectFolder: string = operation.associatedProject?.projectFolder; - const missingFolders: string[] = []; - operation.settings.outputFolderNames.forEach((outputFolderName: string) => { - if (!FileSystem.exists(`${projectFolder}/${outputFolderName}`)) { - missingFolders.push(outputFolderName); + + const { operation } = operationExecutionResult; + + if (cacheAction === CACHE_ACTION_READ) { + const success: boolean = await projectBuildCache.tryRestoreFromCacheAsync(terminal); + if (success) { + ++successCount; + terminal.writeLine( + `Operation "${operation.name}": Outputs have been restored from the build cache."` + ); + terminal.writeLine(`Cache key: ${projectBuildCache.cacheId}`); + } else { + terminal.writeWarningLine( + `Operation "${operation.name}": Outputs could not be restored from the build cache.` + ); + } + } else if (cacheAction === CACHE_ACTION_WRITE) { + if ( + requireOutputFolders && + operation.settings?.outputFolderNames && + operation.settings?.outputFolderNames?.length > 0 + ) { + const projectFolder: string = operation.associatedProject?.projectFolder; + const missingFolders: string[] = []; + operation.settings.outputFolderNames.forEach((outputFolderName: string) => { + if (!FileSystem.exists(`${projectFolder}/${outputFolderName}`)) { + missingFolders.push(outputFolderName); + } + }); + if (missingFolders.length > 0) { + terminal.writeWarningLine( + `Operation "${operation.name}": The following output folders do not exist: "${missingFolders.join('", "')}". Skipping cache population.` + ); + return; + } + } + + const success: boolean = await projectBuildCache.trySetCacheEntryAsync(terminal); + if (success) { + ++successCount; + terminal.writeLine( + `Operation "${operation.name}": Existing outputs have been successfully written to the build cache."` + ); + terminal.writeLine(`Cache key: ${projectBuildCache.cacheId}`); + } else { + terminal.writeErrorLine( + `Operation "${operation.name}": An error occurred while writing existing outputs to the build cache.` + ); } - }); - if (missingFolders.length > 0) { - terminal.writeWarningLine( - `Operation "${operation.name}": The following output folders do not exist: "${missingFolders.join('", "')}". Skipping cache population.` - ); - return; } - } + }, + { concurrency: graph.parallelism } + ); - const success: boolean = await projectBuildCache.trySetCacheEntryAsync(terminal); - if (success) { - ++successCount; - terminal.writeLine( - `Operation "${operation.name}": Existing outputs have been successfully written to the build cache."` - ); - terminal.writeLine(`Cache key: ${projectBuildCache.cacheId}`); - } else { - terminal.writeErrorLine( - `Operation "${operation.name}": An error occurred while writing existing outputs to the build cache.` - ); - } - } - }, - { - concurrency: context.parallelism - } - ); + terminal.writeLine( + `Cache operation "${cacheAction}" completed successfully for ${successCount} out of ${filteredOperations.length} operations.` + ); - terminal.writeLine( - `Cache operation "${cacheAction}" completed successfully for ${successCount} out of ${filteredOperations.size} operations.` + // Bail out with a status indicating success; treat cache read as FromCache. + return cacheAction === CACHE_ACTION_READ ? OperationStatus.FromCache : OperationStatus.Success; + } ); } - ); + }); }); } diff --git a/rush-plugins/rush-buildxl-graph-plugin/src/DropBuildGraphPlugin.ts b/rush-plugins/rush-buildxl-graph-plugin/src/DropBuildGraphPlugin.ts index 8afbd3e812d..1ed5bc705f9 100644 --- a/rush-plugins/rush-buildxl-graph-plugin/src/DropBuildGraphPlugin.ts +++ b/rush-plugins/rush-buildxl-graph-plugin/src/DropBuildGraphPlugin.ts @@ -97,7 +97,7 @@ export class DropBuildGraphPlugin implements IRushPlugin { for (const buildXLCommandName of this._buildXLCommandNames) { session.hooks.runPhasedCommand.for(buildXLCommandName).tap(PLUGIN_NAME, (command: IPhasedCommand) => { - command.hooks.createOperations.tapPromise( + command.hooks.createOperationsAsync.tapPromise( { name: PLUGIN_NAME, stage: Number.MAX_SAFE_INTEGER // Run this after other plugins have created all operations diff --git a/rush-plugins/rush-buildxl-graph-plugin/src/GraphProcessor.ts b/rush-plugins/rush-buildxl-graph-plugin/src/GraphProcessor.ts index 9cf99b5e5d9..fa77ffe99fd 100644 --- a/rush-plugins/rush-buildxl-graph-plugin/src/GraphProcessor.ts +++ b/rush-plugins/rush-buildxl-graph-plugin/src/GraphProcessor.ts @@ -2,7 +2,6 @@ // See LICENSE in the project root for license information. import type { Operation, ILogger } from '@rushstack/rush-sdk'; -import type { ShellOperationRunner } from '@rushstack/rush-sdk/lib/logic/operations/ShellOperationRunner'; import { Colorize } from '@rushstack/terminal'; /** @@ -225,7 +224,7 @@ export class GraphProcessor { package: packageName, dependencies, workingDirectory, - command: (runner as Partial>)?.commandToRun + command: runner?.getConfigHash() }; if (settings?.disableBuildCacheForOperation) { diff --git a/rush-plugins/rush-buildxl-graph-plugin/src/debugGraphFiltering.ts b/rush-plugins/rush-buildxl-graph-plugin/src/debugGraphFiltering.ts index b51f53d2d42..773f444d38f 100644 --- a/rush-plugins/rush-buildxl-graph-plugin/src/debugGraphFiltering.ts +++ b/rush-plugins/rush-buildxl-graph-plugin/src/debugGraphFiltering.ts @@ -22,7 +22,6 @@ const ALLOWED_KEYS: ReadonlySet = new Set([ 'projectFolder', 'dependencies', 'runner', - 'commandToRun', 'isNoOp' ]); @@ -34,7 +33,11 @@ const ALLOWED_KEYS: ReadonlySet = new Set([ * @param depth - the maximum depth to recurse * @param simplify - if true, will replace embedded operations with their operation id */ -export function filterObjectForDebug(obj: object, depth: number = 10, simplify: boolean = false): object { +export function filterObjectForDebug( + obj: object, + depth: number = 10, + simplify: boolean = false +): Record { const output: Record = {}; for (const [key, value] of Object.entries(obj)) { if (BANNED_KEYS.has(key)) { @@ -70,7 +73,11 @@ export function filterObjectForDebug(obj: object, depth: number = 10, simplify: return output; } -export function filterObjectForTesting(obj: object, depth: number = 10, ignoreSets: boolean = false): object { +export function filterObjectForTesting( + obj: object, + depth: number = 10, + ignoreSets: boolean = false +): Record { const output: Record = {}; for (const [key, value] of Object.entries(obj)) { if (!ALLOWED_KEYS.has(key) && !key.match(/^\d+$/)) { diff --git a/rush-plugins/rush-buildxl-graph-plugin/src/examples/debug-graph.json b/rush-plugins/rush-buildxl-graph-plugin/src/examples/debug-graph.json index 702d73cbcb6..34db31aa4a6 100644 --- a/rush-plugins/rush-buildxl-graph-plugin/src/examples/debug-graph.json +++ b/rush-plugins/rush-buildxl-graph-plugin/src/examples/debug-graph.json @@ -5,7 +5,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -19,7 +19,7 @@ { "runner": { "name": "@rushstack/rush-sdk (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -33,7 +33,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -47,7 +47,7 @@ { "runner": { "name": "@rushstack/ts-command-line (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -61,7 +61,7 @@ { "runner": { "name": "@microsoft/rush-lib (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -75,7 +75,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -103,7 +103,7 @@ ], "runner": { "name": "@rushstack/rush-buildxl-graph-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -133,7 +133,7 @@ ], "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -163,7 +163,7 @@ { "runner": { "name": "@rushstack/eslint-patch (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -193,7 +193,7 @@ { "runner": { "name": "@rushstack/eslint-patch (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -207,7 +207,7 @@ { "runner": { "name": "@rushstack/eslint-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -221,7 +221,7 @@ { "runner": { "name": "@rushstack/eslint-plugin-packlets (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -235,7 +235,7 @@ { "runner": { "name": "@rushstack/eslint-plugin-security (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -264,7 +264,7 @@ "dependencies": [], "runner": { "name": "@rushstack/eslint-patch (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -280,7 +280,7 @@ { "runner": { "name": "@rushstack/tree-pattern (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -294,7 +294,7 @@ ], "runner": { "name": "@rushstack/eslint-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -309,7 +309,7 @@ "dependencies": [], "runner": { "name": "@rushstack/tree-pattern (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -325,7 +325,7 @@ { "runner": { "name": "@rushstack/tree-pattern (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -339,7 +339,7 @@ ], "runner": { "name": "@rushstack/eslint-plugin-packlets (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -355,7 +355,7 @@ { "runner": { "name": "@rushstack/tree-pattern (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -369,7 +369,7 @@ ], "runner": { "name": "@rushstack/eslint-plugin-security (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -385,7 +385,7 @@ { "runner": { "name": "@rushstack/lookup-by-path (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -399,7 +399,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -413,7 +413,7 @@ { "runner": { "name": "@rushstack/package-deps-hash (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -427,7 +427,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -441,7 +441,7 @@ { "runner": { "name": "@microsoft/rush-lib (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -455,7 +455,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -483,7 +483,7 @@ { "runner": { "name": "@rushstack/heft-webpack5-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -497,7 +497,7 @@ { "runner": { "name": "@rushstack/stream-collator (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -511,7 +511,7 @@ { "runner": { "name": "@rushstack/ts-command-line (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -525,7 +525,7 @@ { "runner": { "name": "@rushstack/webpack-preserve-dynamic-require-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -539,7 +539,7 @@ ], "runner": { "name": "@rushstack/rush-sdk (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -555,7 +555,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -583,7 +583,7 @@ ], "runner": { "name": "@rushstack/lookup-by-path (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -599,7 +599,7 @@ { "runner": { "name": "@rushstack/heft-config-file (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -613,7 +613,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -627,7 +627,7 @@ { "runner": { "name": "@rushstack/operation-graph (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -641,7 +641,7 @@ { "runner": { "name": "@rushstack/rig-package (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -655,7 +655,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -669,7 +669,7 @@ { "runner": { "name": "@rushstack/ts-command-line (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -683,7 +683,7 @@ { "runner": { "name": "@microsoft/api-extractor (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -711,7 +711,7 @@ ], "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -727,7 +727,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -741,7 +741,7 @@ { "runner": { "name": "@rushstack/rig-package (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -755,7 +755,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -783,7 +783,7 @@ ], "runner": { "name": "@rushstack/heft-config-file (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -813,7 +813,7 @@ ], "runner": { "name": "@rushstack/rig-package (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -829,7 +829,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -857,7 +857,7 @@ ], "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -873,7 +873,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -887,7 +887,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -915,7 +915,7 @@ ], "runner": { "name": "@rushstack/operation-graph (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -931,7 +931,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -959,7 +959,7 @@ ], "runner": { "name": "@rushstack/ts-command-line (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -975,7 +975,7 @@ { "runner": { "name": "@microsoft/api-extractor-model (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -989,7 +989,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1003,7 +1003,7 @@ { "runner": { "name": "@rushstack/rig-package (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1017,7 +1017,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1031,7 +1031,7 @@ { "runner": { "name": "@rushstack/ts-command-line (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1059,7 +1059,7 @@ ], "runner": { "name": "@microsoft/api-extractor (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1075,7 +1075,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1103,7 +1103,7 @@ ], "runner": { "name": "@microsoft/api-extractor-model (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1119,7 +1119,7 @@ { "runner": { "name": "@microsoft/api-extractor (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1147,7 +1147,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1191,7 +1191,7 @@ { "runner": { "name": "@microsoft/api-extractor (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1219,7 +1219,7 @@ { "runner": { "name": "@rushstack/heft-api-extractor-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1233,7 +1233,7 @@ { "runner": { "name": "@rushstack/heft-jest-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1247,7 +1247,7 @@ { "runner": { "name": "@rushstack/heft-lint-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1261,7 +1261,7 @@ { "runner": { "name": "@rushstack/heft-typescript-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1275,7 +1275,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1305,7 +1305,7 @@ { "runner": { "name": "@rushstack/heft-config-file (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1319,7 +1319,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1333,7 +1333,7 @@ { "runner": { "name": "@microsoft/api-extractor (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1361,7 +1361,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1375,7 +1375,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1389,7 +1389,7 @@ ], "runner": { "name": "@rushstack/heft-api-extractor-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1405,7 +1405,7 @@ { "runner": { "name": "@rushstack/heft-config-file (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1419,7 +1419,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1433,7 +1433,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1447,7 +1447,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1475,7 +1475,7 @@ ], "runner": { "name": "@rushstack/heft-jest-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1491,7 +1491,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1519,7 +1519,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1533,7 +1533,7 @@ { "runner": { "name": "@rushstack/heft-typescript-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1547,7 +1547,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1561,7 +1561,7 @@ ], "runner": { "name": "@rushstack/heft-lint-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1577,7 +1577,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1591,7 +1591,7 @@ { "runner": { "name": "@rushstack/heft-config-file (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1619,7 +1619,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1633,7 +1633,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1647,7 +1647,7 @@ ], "runner": { "name": "@rushstack/heft-typescript-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1663,7 +1663,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1677,7 +1677,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1705,7 +1705,7 @@ ], "runner": { "name": "@rushstack/package-deps-hash (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1721,7 +1721,7 @@ { "runner": { "name": "@rushstack/heft-config-file (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1735,7 +1735,7 @@ { "runner": { "name": "@rushstack/lookup-by-path (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1749,7 +1749,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1763,7 +1763,7 @@ { "runner": { "name": "@rushstack/package-deps-hash (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1777,7 +1777,7 @@ { "runner": { "name": "@rushstack/package-extractor (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1791,7 +1791,7 @@ { "runner": { "name": "@rushstack/rig-package (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1805,7 +1805,7 @@ { "runner": { "name": "@rushstack/stream-collator (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1819,7 +1819,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1833,7 +1833,7 @@ { "runner": { "name": "@rushstack/ts-command-line (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1861,7 +1861,7 @@ { "runner": { "name": "@rushstack/heft-webpack5-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1875,7 +1875,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1889,7 +1889,7 @@ { "runner": { "name": "@rushstack/operation-graph (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1903,7 +1903,7 @@ { "runner": { "name": "@rushstack/webpack-deep-imports-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1917,7 +1917,7 @@ { "runner": { "name": "@rushstack/webpack-preserve-dynamic-require-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1931,7 +1931,7 @@ ], "runner": { "name": "@microsoft/rush-lib (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1947,7 +1947,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1961,7 +1961,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -1975,7 +1975,7 @@ { "runner": { "name": "@rushstack/ts-command-line (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2003,7 +2003,7 @@ { "runner": { "name": "@rushstack/heft-webpack5-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2017,7 +2017,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2031,7 +2031,7 @@ { "runner": { "name": "@rushstack/webpack-preserve-dynamic-require-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2045,7 +2045,7 @@ ], "runner": { "name": "@rushstack/package-extractor (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2061,7 +2061,7 @@ { "runner": { "name": "@rushstack/debug-certificate-manager (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2075,7 +2075,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2089,7 +2089,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2103,7 +2103,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2131,7 +2131,7 @@ ], "runner": { "name": "@rushstack/heft-webpack5-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2147,7 +2147,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2161,7 +2161,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2175,7 +2175,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2203,7 +2203,7 @@ ], "runner": { "name": "@rushstack/debug-certificate-manager (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2219,7 +2219,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2247,7 +2247,7 @@ ], "runner": { "name": "@rushstack/webpack-preserve-dynamic-require-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2263,7 +2263,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2277,7 +2277,7 @@ { "runner": { "name": "@rushstack/terminal (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2291,7 +2291,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2319,7 +2319,7 @@ ], "runner": { "name": "@rushstack/stream-collator (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2335,7 +2335,7 @@ { "runner": { "name": "@rushstack/node-core-library (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2363,7 +2363,7 @@ { "runner": { "name": "@rushstack/heft (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", @@ -2377,7 +2377,7 @@ ], "runner": { "name": "@rushstack/webpack-deep-imports-plugin (build)", - "commandToRun": "heft run --only build -- --clean --production" + "_configHash": "heft run --only build -- --clean --production" }, "associatedPhase": { "name": "_phase:build", diff --git a/rush-plugins/rush-buildxl-graph-plugin/src/test/GraphProcessor.test.ts b/rush-plugins/rush-buildxl-graph-plugin/src/test/GraphProcessor.test.ts index ad83eaa7d5f..905d0383af9 100644 --- a/rush-plugins/rush-buildxl-graph-plugin/src/test/GraphProcessor.test.ts +++ b/rush-plugins/rush-buildxl-graph-plugin/src/test/GraphProcessor.test.ts @@ -1,8 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. // See LICENSE in the project root for license information. -import type { IOperationRunner, Operation } from '@rushstack/rush-sdk'; -import type { ShellOperationRunner } from '@rushstack/rush-sdk/lib/logic/operations/ShellOperationRunner'; +import type { IOperationRunner, Operation, OperationStatus } from '@rushstack/rush-sdk'; import { Terminal, NoOpTerminalProvider } from '@rushstack/terminal'; import { GraphProcessor, type IGraphNode } from '../GraphProcessor'; @@ -16,6 +15,41 @@ function sortGraphNodes(graphNodes: IGraphNode[]): IGraphNode[] { return graphNodes.sort((a, b) => (a.id === b.id ? 0 : a.id < b.id ? -1 : 1)); } +class MockRunner implements IOperationRunner { + declare public name: string; + declare public isNoOp: boolean; + declare public silent: boolean; + declare public cacheable: boolean; + declare public reportTiming: boolean; + declare public warningsAreAllowed: boolean; + declare private _configHash: string; + + public async executeAsync(): Promise { + throw new Error('Method not implemented.'); + } + + public getConfigHash(): string { + return this._configHash; + } +} + +function loadDebugGraph(): Operation[] { + const operations: Operation[] = []; + const clonedGraphNodes: typeof debugGraph.OperationMap = JSON.parse( + JSON.stringify(debugGraph.OperationMap) + ); + for (const node of clonedGraphNodes) { + const runner = node.runner; + Object.setPrototypeOf(runner, MockRunner.prototype); + const operation: Operation = { + ...node, + runner + } as unknown as Operation; + operations.push(operation); + } + return operations; +} + describe(GraphProcessor.name, () => { let exampleGraph: readonly IGraphNode[]; let graphParser: GraphProcessor; @@ -39,9 +73,7 @@ describe(GraphProcessor.name, () => { }); it('should process debug-graph.json into graph.json', () => { - let prunedGraph: IGraphNode[] = graphParser.processOperations( - new Set(debugGraph.OperationMap as unknown as Operation[]) - ); + let prunedGraph: IGraphNode[] = graphParser.processOperations(new Set(loadDebugGraph())); prunedGraph = sortGraphNodes(prunedGraph); expect(prunedGraph).toEqual(exampleGraph); @@ -50,7 +82,7 @@ describe(GraphProcessor.name, () => { }); it('should fail if the input schema is invalid', () => { - const clonedOperationMap: Operation[] = JSON.parse(JSON.stringify(debugGraph.OperationMap)); + const clonedOperationMap: Operation[] = loadDebugGraph(); (clonedOperationMap[0].dependencies as unknown as Operation[]).push({ incorrectPhase: { name: 'incorrectPhase' }, incorrectProject: { packageName: 'incorrectProject' } @@ -62,11 +94,9 @@ describe(GraphProcessor.name, () => { }); it('should fail if isNoOp mismatches a command', () => { - const clonedOperationMap: Operation[] = JSON.parse(JSON.stringify(debugGraph.OperationMap)); + const clonedOperationMap: Operation[] = loadDebugGraph(); (clonedOperationMap[0].runner as IOperationRunner & { isNoOp: boolean }).isNoOp = true; - ( - clonedOperationMap[0].runner as unknown as ShellOperationRunner & { commandToRun: string } - ).commandToRun = 'echo "hello world"'; + (clonedOperationMap[0].runner as IOperationRunner).getConfigHash = () => 'echo "hello world"'; const operations: Set = new Set(clonedOperationMap); graphParser.processOperations(operations); expect(emittedErrors).not.toEqual([]); diff --git a/rush-plugins/rush-serve-plugin/README.md b/rush-plugins/rush-serve-plugin/README.md index c8faa8b295d..11180aecaef 100644 --- a/rush-plugins/rush-serve-plugin/README.md +++ b/rush-plugins/rush-serve-plugin/README.md @@ -4,12 +4,13 @@ A Rush plugin that hooks into action execution and runs an express server to ser Supports HTTP/2, compression, CORS, and the new Access-Control-Allow-Private-Network header. -``` +```bash # The user invokes this command $ rush start ``` What happens: + - Rush scans for riggable `rush-serve.json` config files in all projects - Rush uses the configuration in the aforementioned files to configure an Express server to serve project outputs as static (but not cached) content - When a change happens to a source file, Rush's normal watch-mode machinery will rebuild all affected project phases, resulting in new files on disk @@ -22,69 +23,98 @@ This plugin also provides a web socket server that notifies clients of the build The recommended way to connect to the web socket is to serve a static HTML page from the serve plugin using the `globalRouting` configuration. To use the socket: + ```ts import type { IWebSocketEventMessage, IOperationInfo, - IRushSessionInfo, - ReadableOperationStatus + IOperationExecutionState, + ReadableOperationStatus, + IRushSessionInfo } from '@rushstack/rush-serve-plugin/api'; -const socket: WebSocket = new WebSocket(`wss://${self.location.host}${buildStatusWebSocketPath}`); +const socket = new WebSocket(`wss://${self.location.host}${buildStatusWebSocketPath}`); +// Static graph metadata (does not include dynamic status fields) const operationsByName: Map = new Map(); -let buildStatus: ReadableOperationStatus = 'Ready'; +// Current execution state for this iteration +const executionStates: Map = new Map(); +// Queued states for the next iteration (if an iteration has been scheduled but not yet started) +const queuedStates: Map = new Map(); -function updateOperations(operations): void { - for (const operation of operations) { - operationsByName.set(operation.name, operation); - } +let buildStatus: ReadableOperationStatus = 'Ready'; +let sessionInfo: IRushSessionInfo | undefined; - for (const [operationName, operation] of operationsByName) { - // Do something with the operation - } +function upsertOperations(ops: IOperationInfo[]): void { + for (const op of ops) operationsByName.set(op.name, op); +} +function upsertExecutionStates(states: IOperationExecutionState[]): void { + for (const st of states) executionStates.set(st.name, st); } -function updateSessionInfo(sessionInfo: IRushSessionInfo): void { - const { actionName, repositoryIdentifier } = sessionInfo; +function applyQueuedStates(states: IOperationExecutionState[] | undefined): void { + queuedStates.clear(); + if (states) for (const st of states) queuedStates.set(st.name, st); } -function updateBuildStatus(newStatus: ReadableOperationStatus): void { - buildStatus = newStatus; - // Render +function effectiveStatus(name: string): string | undefined { + const exec = executionStates.get(name); + if (exec) return exec.status; + // Optionally fall back to last-known previous iteration results if you track them. + return undefined; } socket.addEventListener('message', (ev) => { - const message: IWebSocketEventMessage = JSON.parse(ev.data); - - switch (message.event) { + const msg: IWebSocketEventMessage = JSON.parse(ev.data as string); + switch (msg.event) { + case 'sync': { + operationsByName.clear(); + executionStates.clear(); + upsertOperations(msg.operations); + upsertExecutionStates(msg.currentExecutionStates); + applyQueuedStates(msg.queuedStates); + sessionInfo = msg.sessionInfo; + buildStatus = msg.status; + break; + } + case 'sync-operations': { + // Static graph changed (e.g. enabled state toggles) – replace definitions only + operationsByName.clear(); + upsertOperations(msg.operations); + break; + } + case 'sync-graph-state': { + // Graph state only – no operation arrays here + break; + } + case 'iteration-scheduled': { + applyQueuedStates(msg.queuedStates); + break; + } case 'before-execute': { - const { operations } = message; - updateOperations(operations); - updateBuildStatus('Executing'); + // Start of an iteration: queuedStates become irrelevant until a new iteration is scheduled + applyQueuedStates(undefined); + upsertExecutionStates(msg.executionStates); + buildStatus = 'Executing'; break; } - case 'status-change': { - const { operations } = message; - updateOperations(operations); + upsertExecutionStates(msg.executionStates); break; } - case 'after-execute': { - const { status } = message; - updateBuildStatus(status); + upsertExecutionStates(msg.executionStates); + buildStatus = msg.status; + // msg.resultByOperation (if present) can be captured for historical display break; } + } - case 'sync': { - operationsByName.clear(); - const { operations, status, sessionInfo } = message; - updateOperations(operations); - updateSessionInfo(sessionInfo); - updateBuildStatus(status); - break; - } + // Example: iterate and render + for (const [name, info] of operationsByName) { + const state = executionStates.get(name); + const status = state?.status ?? '(pending)'; + // renderRow(name, info, status, queuedStates.has(name)); } }); ``` diff --git a/rush-plugins/rush-serve-plugin/package.json b/rush-plugins/rush-serve-plugin/package.json index f61bbab08e5..4fae79b3131 100644 --- a/rush-plugins/rush-serve-plugin/package.json +++ b/rush-plugins/rush-serve-plugin/package.json @@ -22,6 +22,7 @@ "@rushstack/node-core-library": "workspace:*", "@rushstack/rig-package": "workspace:*", "@rushstack/rush-sdk": "workspace:*", + "@rushstack/terminal": "workspace:*", "@rushstack/ts-command-line": "workspace:*", "compression": "~1.7.4", "cors": "~2.8.5", @@ -31,7 +32,6 @@ }, "devDependencies": { "@rushstack/heft": "workspace:*", - "@rushstack/terminal": "workspace:*", "eslint": "~9.37.0", "local-node-rig": "workspace:*", "@types/compression": "~1.7.2", diff --git a/rush-plugins/rush-serve-plugin/src/api.types.ts b/rush-plugins/rush-serve-plugin/src/api.types.ts index 99b5fcb012f..b31d9f32f1b 100644 --- a/rush-plugins/rush-serve-plugin/src/api.types.ts +++ b/rush-plugins/rush-serve-plugin/src/api.types.ts @@ -50,10 +50,13 @@ export interface IOperationInfo { phaseName: string; /** - * If false, this operation is disabled and will/did not execute during the current run. - * The status will be reported as `Skipped`. + * The enabled state of the operation. + * - `never`: The operation is disabled and will not be executed. + * - `ignore-dependency-changes`: The operation will be executed if there are local changes in the project, + * otherwise it will be skipped. + * - `always`: The operation will be executed if it or any dependencies changed. */ - enabled: boolean; + enabled: ReadableOperationEnabledState; /** * If true, this operation is configured to be silent and is included for completeness. @@ -64,6 +67,29 @@ export interface IOperationInfo { * If true, this operation is configured to be a noop and is included for graph completeness. */ noop: boolean; +} + +/** + * Dynamic execution state for an operation (separated from the static graph definition in IOperationInfo). + * Both interfaces contain the operation "name" field for correlation. + */ +export interface IOperationExecutionState { + /** + * The display name of the operation. + */ + name: string; + + /** + * Indicates whether this operation is scheduled to actually run in the current execution iteration. + * This is derived from the scheduler's decision (the execution record's `enabled` boolean), which + * takes into account the configured enabled state plus change detection and dependency invalidation. + */ + runInThisIteration: boolean; + + /** + * If true, this operation currently owns some kind of active resource (e.g. a service or a watch process). + */ + isActive: boolean; /** * The current status of the operation. This value is in PascalCase and is the key of the corresponding `OperationStatus` constant. @@ -102,20 +128,23 @@ export interface IRushSessionInfo { } /** - * Message sent to a WebSocket client at the start of an execution pass. + * Message sent to a WebSocket client at the start of an execution iteration. */ -export interface IWebSocketBeforeExecuteEventMessage { - event: 'before-execute'; - operations: IOperationInfo[]; -} - +// Event (server->client) message interfaces (alphabetically by interface name) /** - * Message sent to a WebSocket client at the end of an execution pass. + * Message sent to a WebSocket client at the end of an execution iteration. */ export interface IWebSocketAfterExecuteEventMessage { event: 'after-execute'; - operations: IOperationInfo[]; + executionStates: IOperationExecutionState[]; status: ReadableOperationStatus; + /** + * Per-operation result records at the time of emission, mirroring the values() of the + * OperationGraph's resultByOperation map. Only present if at least one operation has + * completed execution. Only statuses from actual executions are included; operations + * that were skipped or aborted retain their result from the last time they ran. + */ + resultByOperation?: IOperationExecutionState[]; } /** @@ -125,7 +154,15 @@ export interface IWebSocketAfterExecuteEventMessage { */ export interface IWebSocketBatchStatusChangeEventMessage { event: 'status-change'; - operations: IOperationInfo[]; + executionStates: IOperationExecutionState[]; +} + +/** + * Message sent to a WebSocket client at the start of an execution iteration. + */ +export interface IWebSocketBeforeExecuteEventMessage { + event: 'before-execute'; + executionStates: IOperationExecutionState[]; } /** @@ -135,32 +172,108 @@ export interface IWebSocketBatchStatusChangeEventMessage { */ export interface IWebSocketSyncEventMessage { event: 'sync'; + /** + * Static graph definition (one entry per operation in the graph). + */ operations: IOperationInfo[]; + /** + * Current dynamic execution states for all known operations. + */ + currentExecutionStates: IOperationExecutionState[]; + /** + * Execution states for operations that have been queued for the next iteration (if any) + * when the sync message was generated. + */ + queuedStates?: IOperationExecutionState[]; sessionInfo: IRushSessionInfo; status: ReadableOperationStatus; + graphState: { + parallelism: number; + debugMode: boolean; + verbose: boolean; + pauseNextIteration: boolean; + status: ReadableOperationStatus; + hasScheduledIteration: boolean; + }; + /** + * Per-operation result records at the time of emission, mirroring the values() of the + * OperationGraph's resultByOperation map. Only present if at least one operation has + * completed execution. Only statuses from actual executions are included; operations + * that were skipped or aborted retain their result from the last time they ran. + */ + resultByOperation?: IOperationExecutionState[]; +} + +/** + * Message sent to a WebSocket client containing a full refresh of only the dynamic execution states. + */ +export interface IWebSocketSyncOperationsEventMessage { + event: 'sync-operations'; + operations: IOperationInfo[]; +} + +/** + * Message sent when an iteration is queued with its initial set of queued operations. + */ +export interface IWebSocketPassQueuedEventMessage { + event: 'iteration-scheduled'; + queuedStates: IOperationExecutionState[]; +} + +/** + * Message sent to a WebSocket client containing only updated settings (no operations list). + */ +export interface IWebSocketSyncGraphStateEventMessage { + event: 'sync-graph-state'; + graphState: IWebSocketSyncEventMessage['graphState']; +} + +export interface IWebSocketTerminalChunkEventMessage { + event: 'terminal-chunk'; + kind: 'stdout' | 'stderr'; + text: string; } /** * The set of possible messages sent to a WebSocket client. */ export type IWebSocketEventMessage = - | IWebSocketBeforeExecuteEventMessage | IWebSocketAfterExecuteEventMessage | IWebSocketBatchStatusChangeEventMessage - | IWebSocketSyncEventMessage; + | IWebSocketBeforeExecuteEventMessage + | IWebSocketSyncEventMessage + | IWebSocketSyncOperationsEventMessage + | IWebSocketPassQueuedEventMessage + | IWebSocketSyncGraphStateEventMessage; +// Command (client->server) message interfaces (alphabetically by interface name) /** - * Message received from a WebSocket client to request a sync. + * Message received from a WebSocket client to request abortion of the current execution iteration. */ -export interface IWebSocketSyncCommandMessage { - command: 'sync'; +export interface IWebSocketAbortExecutionCommandMessage { + command: 'abort-execution'; } /** - * Message received from a WebSocket client to request abortion of the current execution pass. + * Message to abort the entire watch session (similar to pressing 'q'). */ -export interface IWebSocketAbortExecutionCommandMessage { - command: 'abort-execution'; +export interface IWebSocketAbortSessionCommandMessage { + command: 'abort-session'; +} + +/** + * Message received from a WebSocket client to request closing of active operation runners. + */ +export interface IWebSocketCloseRunnersCommandMessage { + command: 'close-runners'; + operationNames?: string[]; +} + +/** + * Message received from a WebSocket client to request execution of a new execution iteration. + */ +export interface IWebSocketExecuteCommandMessage { + command: 'execute'; } /** @@ -168,27 +281,87 @@ export interface IWebSocketAbortExecutionCommandMessage { */ export interface IWebSocketInvalidateCommandMessage { command: 'invalidate'; - operationNames: string[]; + operationNames?: string[]; } /** - * The set of possible operation enabled states. + * Message received from a WebSocket client to toggle debug logging mode. + * A value of true enables debug mode; false disables it. */ -export type OperationEnabledState = 'never' | 'changed' | 'affected' | 'default'; +export interface IWebSocketSetDebugCommandMessage { + command: 'set-debug'; + value: boolean; +} /** - * Message received from a WebSocket client to change the enabled states of operations. + * Message received from a WebSocket client to change the enabled states of one or more operations. */ export interface IWebSocketSetEnabledStatesCommandMessage { command: 'set-enabled-states'; - enabledStateByOperationName: Record; + /** + * The names of the operations whose enabled state should be updated. + */ + operationNames: string[]; + /** + * The target enabled state. 'never', 'ignore-dependency-changes', or 'affected'. + */ + targetState: ReadableOperationEnabledState; + /** + * Mode controlling how the enabled state is applied. "safe" applies dependency-aware logic, + * "unsafe" only mutates the provided operations. + */ + mode: 'safe' | 'unsafe'; +} + +/** + * Message received to set absolute parallelism value. + */ +export interface IWebSocketSetParallelismCommandMessage { + command: 'set-parallelism'; + parallelism: number; } +/** + * Message received from a WebSocket client to set whether new execution iterations are paused when scheduled. + * A value of true means iterations are paused (manual mode); false means iterations run automatically. + */ +export interface IWebSocketSetPauseNextIterationCommandMessage { + command: 'set-pause-next-iteration'; + value: boolean; +} + +/** + * Message received from a WebSocket client to set verbose logging mode (true => verbose on, quiet off). + */ +export interface IWebSocketSetVerboseCommandMessage { + command: 'set-verbose'; + value: boolean; // true => verbose on (quiet off) +} + +/** + * Message received from a WebSocket client to request a sync of the full state. + */ +export interface IWebSocketSyncCommandMessage { + command: 'sync'; +} + +/** + * The set of possible operation enabled states. + */ +export type ReadableOperationEnabledState = 'never' | 'ignore-dependency-changes' | 'affected'; + /** * The set of possible messages received from a WebSocket client. */ export type IWebSocketCommandMessage = - | IWebSocketSyncCommandMessage | IWebSocketAbortExecutionCommandMessage + | IWebSocketAbortSessionCommandMessage + | IWebSocketCloseRunnersCommandMessage + | IWebSocketExecuteCommandMessage | IWebSocketInvalidateCommandMessage - | IWebSocketSetEnabledStatesCommandMessage; + | IWebSocketSetDebugCommandMessage + | IWebSocketSetEnabledStatesCommandMessage + | IWebSocketSetParallelismCommandMessage + | IWebSocketSetPauseNextIterationCommandMessage + | IWebSocketSetVerboseCommandMessage + | IWebSocketSyncCommandMessage; diff --git a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts index 4c76680f170..a135c7aec04 100644 --- a/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts +++ b/rush-plugins/rush-serve-plugin/src/phasedCommandHandler.ts @@ -54,13 +54,13 @@ export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions const webSocketServerUpgrader: WebSocketServerUpgrader | undefined = tryEnableBuildStatusWebSocketServer(options); - command.hooks.createOperations.tapPromise( + command.hooks.createOperationsAsync.tapPromise( { name: PLUGIN_NAME, stage: -1 }, async (operations: Set, context: ICreateOperationsContext) => { - if (!context.isInitial || !context.isWatch) { + if (!context.isWatch) { return operations; } @@ -124,19 +124,21 @@ export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions app.use(compression({})); - const selectedProjects: ReadonlySet = context.projectSelection; + const relevantProjects: ReadonlySet = context.generateFullGraph + ? new Set(context.rushConfiguration.projects) + : expandAllDependencies(context.projectSelection); const serveConfig: RushServeConfiguration = new RushServeConfiguration(); const routingRules: IRoutingRule[] = await serveConfig.loadProjectConfigsAsync( - selectedProjects, + relevantProjects, logger.terminal, globalRoutingRules ); const { logServePath } = options; if (logServePath) { - for (const project of selectedProjects) { + for (const project of relevantProjects) { const projectLogServePath: string = getLogServePathForProject(logServePath, project.packageName); routingRules.push({ @@ -239,9 +241,23 @@ export async function phasedCommandHandler(options: IPhasedCommandHandlerOptions (portParameter as unknown as { _value?: string })._value = `${activePort}`; } + logHost(); + return operations; } ); - command.hooks.waitingForChanges.tap(PLUGIN_NAME, logHost); + command.hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph) => { + graph.hooks.onIdle.tap(PLUGIN_NAME, logHost); + }); +} + +function expandAllDependencies(projects: Iterable): Set { + const expanded: Set = new Set(projects); + for (const project of expanded) { + for (const dependency of project.dependencyProjects) { + expanded.add(dependency); + } + } + return expanded; } diff --git a/rush-plugins/rush-serve-plugin/src/tryEnableBuildStatusWebSocketServer.ts b/rush-plugins/rush-serve-plugin/src/tryEnableBuildStatusWebSocketServer.ts index 1ecf5f10063..78a82c30eee 100644 --- a/rush-plugins/rush-serve-plugin/src/tryEnableBuildStatusWebSocketServer.ts +++ b/rush-plugins/rush-serve-plugin/src/tryEnableBuildStatusWebSocketServer.ts @@ -13,24 +13,28 @@ import { type IOperationExecutionResult, OperationStatus, type ILogFilePaths, - type ICreateOperationsContext, - type IExecutionResult, type RushConfiguration, - type IExecuteOperationsContext + type IOperationGraph, + type IOperationGraphIterationOptions } from '@rushstack/rush-sdk'; +import { type ITerminalChunk, TerminalChunkKind, TerminalWritable } from '@rushstack/terminal'; import type { ReadableOperationStatus, ILogFileURLs, IOperationInfo, + IOperationExecutionState, IWebSocketEventMessage, IRushSessionInfo, IWebSocketSyncEventMessage, - OperationEnabledState, IWebSocketBeforeExecuteEventMessage, IWebSocketAfterExecuteEventMessage, IWebSocketBatchStatusChangeEventMessage, - IWebSocketCommandMessage + IWebSocketCommandMessage, + IWebSocketPassQueuedEventMessage, + IWebSocketSyncOperationsEventMessage, + IWebSocketTerminalChunkEventMessage, + IWebSocketSyncGraphStateEventMessage } from './api.types'; import { PLUGIN_NAME } from './constants'; import type { IPhasedCommandHandlerOptions } from './types'; @@ -67,6 +71,27 @@ export function getLogServePathForProject(logServePath: string, packageName: str return `${logServePath}/${packageName}`; } +export class WebSocketTerminalWritable extends TerminalWritable { + private _webSockets: ReadonlySet; + + public constructor(webSockets: ReadonlySet) { + super(); + this._webSockets = webSockets; + } + + protected override onWriteChunk(chunk: ITerminalChunk): void { + const message: IWebSocketTerminalChunkEventMessage = { + event: 'terminal-chunk', + kind: chunk.kind === TerminalChunkKind.Stderr ? 'stderr' : 'stdout', + text: chunk.text + }; + const stringifiedMessage: string = JSON.stringify(message); + for (const ws of this._webSockets) { + ws.send(stringifiedMessage); + } + } +} + /** * If the `buildStatusWebSocketPath` option is configured, this function returns a `WebSocketServerUpgrader` callback * that can be used to add a WebSocket server to the HTTPS server. The WebSocket server sends messages @@ -83,7 +108,6 @@ export function tryEnableBuildStatusWebSocketServer( const operationStates: Map = new Map(); let buildStatus: ReadableOperationStatus = 'Ready'; - let executionAbortController: AbortController | undefined; const webSockets: Set = new Set(); @@ -129,8 +153,7 @@ export function tryEnableBuildStatusWebSocketServer( /** * Maps the internal Rush record down to a subset that is JSON-friendly and human readable. */ - function convertToOperationInfo(record: IOperationExecutionResult): IOperationInfo | undefined { - const { operation } = record; + function convertToOperationInfo(operation: Operation): IOperationInfo | undefined { const { name, associatedPhase, associatedProject, runner, enabled } = operation; if (!name || !runner) { @@ -144,32 +167,58 @@ export function tryEnableBuildStatusWebSocketServer( dependencies: Array.from(operation.dependencies, (dep) => dep.name), packageName, phaseName: associatedPhase.name, - - enabled, + enabled: + enabled === false + ? 'never' + : enabled === 'ignore-dependency-changes' + ? 'ignore-dependency-changes' + : 'affected', silent: runner.silent, - noop: !!runner.isNoOp, + noop: !!runner.isNoOp + }; + } + function convertToExecutionState(record: IOperationExecutionResult): IOperationExecutionState | undefined { + const { operation } = record; + const { name, associatedProject, runner } = operation; + if (!name || !runner) return; + const { packageName } = associatedProject; + return { + name, + runInThisIteration: record.enabled, + isActive: !!runner.isActive, status: readableStatusFromStatus[record.status], startTime: record.stopwatch.startTime, endTime: record.stopwatch.endTime, - logFileURLs: convertToLogFileUrls(record.logFilePaths, packageName) }; } - function convertToOperationInfoArray(records: Iterable): IOperationInfo[] { - const operations: IOperationInfo[] = []; + function convertToOperationInfoArray(operations: Iterable): IOperationInfo[] { + const infos: IOperationInfo[] = []; - for (const record of records) { - const info: IOperationInfo | undefined = convertToOperationInfo(record); + for (const operation of operations) { + const info: IOperationInfo | undefined = convertToOperationInfo(operation); if (info) { - operations.push(info); + infos.push(info); } } - Sort.sortBy(operations, (x) => x.name); - return operations; + Sort.sortBy(infos, (x) => x.name); + return infos; + } + + function convertToExecutionStateArray( + records: Iterable + ): IOperationExecutionState[] { + const states: IOperationExecutionState[] = []; + for (const record of records) { + const state: IOperationExecutionState | undefined = convertToExecutionState(record); + if (state) states.push(state); + } + Sort.sortBy(states, (x) => x.name); + return states; } function sendWebSocketMessage(message: IWebSocketEventMessage): void { @@ -185,115 +234,145 @@ export function tryEnableBuildStatusWebSocketServer( repositoryIdentifier: getRepositoryIdentifier(options.rushConfiguration) }; + let lastGraph: IOperationGraph | undefined; + // Operations that have been queued for an upcoming execution iteration (captured at queue time) + let queuedStates: IOperationExecutionResult[] | undefined; + + function getGraphStateSnapshot(): IWebSocketSyncEventMessage['graphState'] | undefined { + if (!lastGraph) return; + return { + parallelism: lastGraph.parallelism, + debugMode: lastGraph.debugMode, + verbose: !lastGraph.quietMode, + pauseNextIteration: lastGraph.pauseNextIteration, + status: buildStatus, + hasScheduledIteration: lastGraph.hasScheduledIteration + }; + } + function sendSyncMessage(webSocket: WebSocket): void { + const records: Set = new Set(operationStates?.values() ?? []); const syncMessage: IWebSocketSyncEventMessage = { event: 'sync', - operations: convertToOperationInfoArray(operationStates?.values() ?? []), + operations: convertToOperationInfoArray(lastGraph?.operations ?? []), + currentExecutionStates: convertToExecutionStateArray(records), + queuedStates: queuedStates ? convertToExecutionStateArray(queuedStates) : undefined, sessionInfo, - status: buildStatus + status: buildStatus, + graphState: getGraphStateSnapshot() ?? { + parallelism: 1, + debugMode: false, + verbose: true, + pauseNextIteration: false, + status: buildStatus, + hasScheduledIteration: false + }, + resultByOperation: lastGraph + ? convertToExecutionStateArray(lastGraph.resultByOperation.values()) + : undefined }; - webSocket.send(JSON.stringify(syncMessage)); } - const { hooks } = command; - - let invalidateOperation: ((operation: Operation, reason: string) => void) | undefined; - - const operationEnabledStates: Map = new Map(); - hooks.createOperations.tap( - { - name: PLUGIN_NAME, - stage: Infinity - }, - (operations: Set, context: ICreateOperationsContext) => { - const potentiallyAffectedOperations: Set = new Set(); - for (const operation of operations) { - const { associatedProject } = operation; - if (context.projectsInUnknownState.has(associatedProject)) { - potentiallyAffectedOperations.add(operation); - } - } - for (const operation of potentiallyAffectedOperations) { - for (const consumer of operation.consumers) { - potentiallyAffectedOperations.add(consumer); + command.hooks.onGraphCreatedAsync.tap(PLUGIN_NAME, (graph, context) => { + lastGraph = graph; + const { hooks } = graph; + + graph.addTerminalDestination(new WebSocketTerminalWritable(webSockets)); + + hooks.beforeExecuteIterationAsync.tap( + PLUGIN_NAME, + ( + operationsToExecute: ReadonlyMap, + iterationOptions: IOperationGraphIterationOptions + ): void => { + // Clear queuedStates when the iteration begins executing + queuedStates = undefined; + for (const [operation, result] of operationsToExecute) { + operationStates.set(operation.name, result); } - const { name } = operation; - const expectedState: OperationEnabledState | undefined = operationEnabledStates.get(name); - switch (expectedState) { - case 'affected': - operation.enabled = true; - break; - case 'never': - operation.enabled = false; - break; - case 'changed': - operation.enabled = context.projectsInUnknownState.has(operation.associatedProject); - break; - case 'default': - case undefined: - // Use the original value. - break; - } + const beforeExecuteMessage: IWebSocketBeforeExecuteEventMessage = { + event: 'before-execute', + executionStates: convertToExecutionStateArray(operationsToExecute.values()) + }; + buildStatus = 'Executing'; + sendWebSocketMessage(beforeExecuteMessage); } - - invalidateOperation = context.invalidateOperation; - - return operations; - } - ); - - hooks.beforeExecuteOperations.tap( - PLUGIN_NAME, - ( - operationsToExecute: Map, - context: IExecuteOperationsContext - ): void => { - for (const [operation, result] of operationsToExecute) { - operationStates.set(operation.name, result); + ); + + hooks.afterExecuteIterationAsync.tap( + PLUGIN_NAME, + ( + status: OperationStatus, + operationResults: ReadonlyMap + ): OperationStatus => { + buildStatus = readableStatusFromStatus[status]; + const states: IOperationExecutionState[] = convertToExecutionStateArray( + operationResults.values() ?? [] + ); + const afterExecuteMessage: IWebSocketAfterExecuteEventMessage = { + event: 'after-execute', + executionStates: states, + status: buildStatus, + resultByOperation: lastGraph + ? convertToExecutionStateArray(lastGraph.resultByOperation.values()) + : undefined + }; + sendWebSocketMessage(afterExecuteMessage); + return status; } + ); + + // Batched operation state updates + hooks.onExecutionStatesUpdated.tap( + PLUGIN_NAME, + (records: ReadonlySet): void => { + const states: IOperationExecutionState[] = convertToExecutionStateArray(records.values()); + const message: IWebSocketBatchStatusChangeEventMessage = { + event: 'status-change', + executionStates: states + }; + sendWebSocketMessage(message); + } + ); + + // Capture queued operations for next iteration + hooks.onIterationScheduled.tap( + PLUGIN_NAME, + (queuedMap: ReadonlyMap): void => { + queuedStates = Array.from(queuedMap.values()); + const message: IWebSocketPassQueuedEventMessage = { + event: 'iteration-scheduled', + queuedStates: convertToExecutionStateArray(queuedStates) + }; + sendWebSocketMessage(message); + } + ); + + // Broadcast graph state changes + hooks.onGraphStateChanged.tap(PLUGIN_NAME, () => { + const graphState: IWebSocketSyncEventMessage['graphState'] | undefined = getGraphStateSnapshot(); + if (graphState) { + const message: IWebSocketSyncGraphStateEventMessage = { + event: 'sync-graph-state', + graphState + }; + sendWebSocketMessage(message); + // Execution state may depend on graph properties, so broadcast states. + } + }); - executionAbortController = context.abortController; - - const beforeExecuteMessage: IWebSocketBeforeExecuteEventMessage = { - event: 'before-execute', - operations: convertToOperationInfoArray(operationsToExecute.values()) + // Broadcast enabled state changes (full operations sync for simplicity) + // When enable states change, emit a lightweight sync-operations message conveying the static graph changes. + // The client will preserve existing dynamic state arrays. + hooks.onEnableStatesChanged.tap(PLUGIN_NAME, () => { + const operationsMessage: IWebSocketSyncOperationsEventMessage = { + event: 'sync-operations', + operations: convertToOperationInfoArray(lastGraph?.operations ?? []) }; - buildStatus = 'Executing'; - sendWebSocketMessage(beforeExecuteMessage); - } - ); - - hooks.afterExecuteOperations.tap(PLUGIN_NAME, (result: IExecutionResult): void => { - buildStatus = readableStatusFromStatus[result.status]; - const infos: IOperationInfo[] = convertToOperationInfoArray(result.operationResults.values() ?? []); - const afterExecuteMessage: IWebSocketAfterExecuteEventMessage = { - event: 'after-execute', - operations: infos, - status: buildStatus - }; - sendWebSocketMessage(afterExecuteMessage); - }); - - const pendingStatusChanges: Map = new Map(); - let statusChangeTimeout: NodeJS.Immediate | undefined; - function sendBatchedStatusChange(): void { - statusChangeTimeout = undefined; - const infos: IOperationInfo[] = convertToOperationInfoArray(pendingStatusChanges.values()); - pendingStatusChanges.clear(); - const message: IWebSocketBatchStatusChangeEventMessage = { - event: 'status-change', - operations: infos - }; - sendWebSocketMessage(message); - } - - hooks.onOperationStatusChanged.tap(PLUGIN_NAME, (record: IOperationExecutionResult): void => { - pendingStatusChanges.set(record.operation, record); - if (!statusChangeTimeout) { - statusChangeTimeout = setImmediate(sendBatchedStatusChange); - } + sendWebSocketMessage(operationsMessage); + }); }); const connector: WebSocketServerUpgrader = (server: Http2SecureServer) => { @@ -301,10 +380,35 @@ export function tryEnableBuildStatusWebSocketServer( server: server as unknown as HTTPSecureServer, path: buildStatusWebSocketPath }); + + command.sessionAbortController.signal.addEventListener( + 'abort', + () => { + wss.close(); + webSockets.forEach((ws) => ws.close()); + }, + { once: true } + ); + + function namesToOperations(operationNames?: string[]): Operation[] | undefined { + if (!operationNames || !lastGraph) { + return; + } + + const operationNameSet: Set = new Set(operationNames); + const namedOperations: Operation[] = []; + for (const operation of lastGraph.operations) { + if (operationNameSet.has(operation.name)) { + namedOperations.push(operation); + } + } + return namedOperations; + } + wss.addListener('connection', (webSocket: WebSocket): void => { webSockets.add(webSocket); - sendSyncMessage(webSocket); + sendSyncMessage(webSocket); // includes settings webSocket.addEventListener('message', (ev: MessageEvent) => { const parsedMessage: IWebSocketCommandMessage = JSON.parse(ev.data.toString()); @@ -315,31 +419,79 @@ export function tryEnableBuildStatusWebSocketServer( } case 'set-enabled-states': { - const { enabledStateByOperationName } = parsedMessage; - for (const [name, state] of Object.entries(enabledStateByOperationName)) { - operationEnabledStates.set(name, state); + if (lastGraph) { + const { operationNames, targetState, mode } = parsedMessage; + const operations: Operation[] | undefined = namesToOperations(operationNames); + if (operations && operations.length) { + lastGraph.setEnabledStates( + operations, + targetState === 'ignore-dependency-changes' ? targetState : targetState !== 'never', + mode + ); + } } break; } case 'invalidate': { const { operationNames } = parsedMessage; - const operationNameSet: Set = new Set(operationNames); - if (invalidateOperation) { - for (const operationName of operationNameSet) { - const operationState: IOperationExecutionResult | undefined = - operationStates.get(operationName); - if (operationState) { - invalidateOperation(operationState.operation, 'Invalidated via WebSocket'); - operationStates.delete(operationName); - } - } + if (lastGraph) { + const operations: Iterable | undefined = namesToOperations(operationNames); + lastGraph.invalidateOperations(operations, 'manual-invalidation'); } break; } case 'abort-execution': { - executionAbortController?.abort(); + void lastGraph?.abortCurrentIterationAsync(); + break; + } + + case 'close-runners': { + const { operationNames } = parsedMessage; + if (lastGraph) { + const operations: Operation[] | undefined = namesToOperations(operationNames); + void lastGraph.closeRunnersAsync(operations); + } + break; + } + + case 'execute': { + if (lastGraph) { + const definedExecutionManager: IOperationGraph = lastGraph; + void definedExecutionManager.scheduleIterationAsync({}).then(() => { + return definedExecutionManager.executeScheduledIterationAsync(); + }); + } + break; + } + + case 'set-debug': { + if (lastGraph) lastGraph.debugMode = !!parsedMessage.value; + break; + } + + case 'set-verbose': { + if (lastGraph) lastGraph.quietMode = !parsedMessage.value; // invert + break; + } + + case 'set-pause-next-iteration': { + if (lastGraph && typeof parsedMessage.value === 'boolean') { + lastGraph.pauseNextIteration = parsedMessage.value; + } + break; + } + + case 'set-parallelism': { + if (lastGraph && typeof parsedMessage.parallelism === 'number') { + lastGraph.parallelism = parsedMessage.parallelism; + } + break; + } + + case 'abort-session': { + command.sessionAbortController.abort(); break; }