From a49a606ac33344ac38e133d1a9a5ba57cd85b3f0 Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Tue, 24 Mar 2026 15:11:56 -0600 Subject: [PATCH 1/5] fix: first pass at exit codes --- messages/agent.test.results.md | 9 ++++ messages/agent.test.resume.md | 9 ++++ messages/agent.test.run-eval.md | 9 ++++ messages/agent.test.run.md | 9 ++++ messages/agent.validate.authoring-bundle.md | 11 +++++ src/commands/agent/test/resume.ts | 6 +++ src/commands/agent/test/run-eval.ts | 5 ++- src/commands/agent/test/run.ts | 6 +++ test/nuts/agent.test.run-eval.nut.ts | 50 +++++++++++---------- 9 files changed, 88 insertions(+), 26 deletions(-) diff --git a/messages/agent.test.results.md b/messages/agent.test.results.md index d3b1d0b4..68008c6c 100644 --- a/messages/agent.test.results.md +++ b/messages/agent.test.results.md @@ -8,6 +8,15 @@ This command requires a job ID, which the original "agent test run" command disp By default, this command outputs test results in human-readable tables for each test case. The tables show whether the test case passed, the expected and actual values, the test score, how long the test took, and more. Use the --result-format to display the test results in JSON or Junit format. Use the --output-dir flag to write the results to a file rather than to the terminal. +ERROR CODES + + Succeeded (0) Results retrieved successfully. Test results (passed/failed) are in the output. + Failed (1) Command couldn't execute due to invalid job ID, API errors, network issues, or system errors. + +ENVIRONMENT VARIABLES + + SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. + # flags.job-id.summary Job ID of the completed agent test run. diff --git a/messages/agent.test.resume.md b/messages/agent.test.resume.md index cf0cfcfc..ef0feff4 100644 --- a/messages/agent.test.resume.md +++ b/messages/agent.test.resume.md @@ -10,6 +10,15 @@ Use the --wait flag to specify the number of minutes for this command to wait fo By default, this command outputs test results in human-readable tables for each test case. The tables show whether the test case passed, the expected and actual values, the test score, how long the test took, and more. Use the --result-format to display the test results in JSON or Junit format. Use the --output-dir flag to write the results to a file rather than to the terminal. +ERROR CODES + + Succeeded (0) Test completed successfully. Test results (passed/failed) are in the JSON output. + Failed (1) Command couldn't execute due to invalid job ID, API errors, network issues, or system errors. Exit code 1 also indicates tests encountered execution errors. + +ENVIRONMENT VARIABLES + + SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. + # flags.job-id.summary Job ID of the original agent test run. diff --git a/messages/agent.test.run-eval.md b/messages/agent.test.run-eval.md index 304ab4bd..e1d6df12 100644 --- a/messages/agent.test.run-eval.md +++ b/messages/agent.test.run-eval.md @@ -12,6 +12,15 @@ When you provide a JSON payload, it's sent directly to the API with optional nor Supports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions, semantic similarity scoring, and LLM-based quality ratings. +ERROR CODES + + Succeeded (0) Tests completed successfully. Test results (passed/failed) are in the JSON output. + Failed (1) Execution error occurred. Tests couldn't run due to API errors, network issues, invalid parameters, or system errors. + +ENVIRONMENT VARIABLES + + SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. + # flags.spec.summary Path to test spec file (YAML or JSON). Supports reading from stdin when piping content. diff --git a/messages/agent.test.run.md b/messages/agent.test.run.md index 09adc433..4cfb2c15 100644 --- a/messages/agent.test.run.md +++ b/messages/agent.test.run.md @@ -10,6 +10,15 @@ By default, this command starts the agent test in your org, but it doesn't wait By default, this command outputs test results in human-readable tables for each test case, if the test completes in time. The tables show whether the test case passed, the expected and actual values, the test score, how long the test took, and more. Use the --result-format to display the test results in JSON or Junit format. Use the --output-dir flag to write the results to a file rather than to the terminal. +ERROR CODES + + Succeeded (0) Test started successfully (without --wait), or test completed successfully (with --wait). + Failed (1) Command couldn't execute due to API errors, network issues, invalid test name, or system errors. When using --wait, exit code 1 also indicates tests encountered execution errors. + +ENVIRONMENT VARIABLES + + SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. + # flags.api-name.summary API name of the agent test to run; corresponds to the name of the AiEvaluationDefinition metadata component that implements the agent test. diff --git a/messages/agent.validate.authoring-bundle.md b/messages/agent.validate.authoring-bundle.md index 0a1ac99c..2c84291b 100644 --- a/messages/agent.validate.authoring-bundle.md +++ b/messages/agent.validate.authoring-bundle.md @@ -10,6 +10,17 @@ This command validates that the Agent Script file in the authoring bundle compil This command uses the API name of the authoring bundle. If you don't provide an API name with the --api-name flag, the command searches the current DX project and outputs a list of authoring bundles that it found for you to choose from. +ERROR CODES + + Succeeded (0) Agent Script file compiled successfully without errors. + Failed (1) Compilation errors found in the Agent Script file. + NotFound (2) Validation/compilation API returned HTTP 404. The API endpoint may not be available in your org or region. + ServerError (3) Validation/compilation API returned HTTP 500. A server error occurred during compilation. + +ENVIRONMENT VARIABLES + + SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. + # examples - Validate an authoring bundle by being prompted for its API name; use your default org: diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index 65a40269..7580ac49 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -85,6 +85,12 @@ export default class AgentTestResume extends SfCommand { verbose: flags.verbose, }); + // Set exit code to 1 only for execution errors (tests couldn't run properly) + // Test assertion failures are business logic and should not affect exit code + if (response?.testCases.some((tc) => tc.status === 'ERROR')) { + process.exitCode = 1; + } + return { ...response!, runId, status: 'COMPLETED' }; } } diff --git a/src/commands/agent/test/run-eval.ts b/src/commands/agent/test/run-eval.ts index f364c434..3da15d75 100644 --- a/src/commands/agent/test/run-eval.ts +++ b/src/commands/agent/test/run-eval.ts @@ -272,8 +272,9 @@ export default class AgentTestRunEval extends SfCommand { // 10. Build structured result for --json const { summary, testSummaries } = buildResultSummary(mergedResponse); - // Set exit code to 1 if any tests failed - if (summary.failed > 0 || summary.errors > 0) { + // Set exit code to 1 only for execution errors (tests couldn't run) + // Test failures (assertions failed) are business logic and should not affect exit code + if (summary.errors > 0) { process.exitCode = 1; } diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 9376965e..11f24d83 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -129,6 +129,12 @@ export default class AgentTestRun extends SfCommand { verbose: flags.verbose, }); + // Set exit code to 1 only for execution errors (tests couldn't run properly) + // Test assertion failures are business logic and should not affect exit code + if (detailsResponse?.testCases.some((tc) => tc.status === 'ERROR')) { + process.exitCode = 1; + } + return { ...detailsResponse!, status: 'COMPLETED', runId: response.runId }; } else { this.mso.stop(); diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index e21d4b30..ad5f83ad 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -39,8 +39,8 @@ describe('agent test run-eval', function () { describe('run-eval with JSON file', () => { it('should run evaluation with JSON payload file', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail (business logic), unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; expect(output?.result).to.be.ok; expect(output?.result.tests).to.be.an('array'); @@ -50,12 +50,14 @@ describe('agent test run-eval', function () { expect(output?.result.summary.failed).to.be.a('number'); expect(output?.result.summary.scored).to.be.a('number'); expect(output?.result.summary.errors).to.be.a('number'); + // Verify no execution errors (only test failures are acceptable) + expect(output?.result.summary.errors).to.equal(0); }); it('should run evaluation with normalized payload', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail, unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; expect(output?.result.tests[0]).to.be.ok; expect(output?.result.tests[0].id).to.equal('test-topic-routing'); @@ -67,8 +69,8 @@ describe('agent test run-eval', function () { describe('run-eval with YAML file', () => { it('should run evaluation with YAML test spec file', async () => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail, unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; expect(output?.result).to.be.ok; expect(output?.result.tests).to.be.an('array'); @@ -78,8 +80,8 @@ describe('agent test run-eval', function () { it('should auto-infer agent name from YAML subjectName', async () => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail, unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; // Should succeed without explicit --api-name flag expect(output?.result).to.be.ok; @@ -88,8 +90,8 @@ describe('agent test run-eval', function () { it('should handle YAML spec with contextVariables', async () => { const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail, unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; // Verify the command succeeds with contextVariables expect(output?.result).to.be.ok; @@ -102,8 +104,8 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail, unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; expect(output?.result).to.be.ok; expect(output?.result.tests).to.be.an('array'); @@ -111,30 +113,30 @@ describe('agent test run-eval', function () { it('should use custom batch size', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail, unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; expect(output?.result).to.be.ok; expect(output?.result.tests).to.be.an('array'); }); it('should support different result formats', async () => { - // Test human format (default) - don't enforce exit code since tests may fail + // Test human format (default) - exit code 0 even if tests fail const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; - const humanOutput = execCmd(humanCommand).shellOutput.stdout; + const humanOutput = execCmd(humanCommand, { ensureExitCode: 0 }).shellOutput.stdout; expect(humanOutput).to.be.ok; expect(humanOutput).to.be.a('string'); - // Test tap format - don't enforce exit code since tests may fail + // Test tap format - exit code 0 even if tests fail const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; - const tapOutput = execCmd(tapCommand).shellOutput.stdout; + const tapOutput = execCmd(tapCommand, { ensureExitCode: 0 }).shellOutput.stdout; expect(tapOutput).to.include('TAP version'); - // Test junit format - don't enforce exit code since tests may fail + // Test junit format - exit code 0 even if tests fail const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; - const junitOutput = execCmd(junitCommand).shellOutput.stdout; + const junitOutput = execCmd(junitCommand, { ensureExitCode: 0 }).shellOutput.stdout; expect(junitOutput).to.include(' { it('should include test summaries with correct structure', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail, unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; expect(output?.result.tests).to.be.an('array'); const firstTest = output?.result.tests[0]; @@ -198,8 +200,8 @@ describe('agent test run-eval', function () { it('should include summary with all metrics', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + // Exit code should be 0 even if tests fail, unless there are execution errors + const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; const summary = output?.result.summary; expect(summary).to.have.property('passed'); From b6d5eaa722dce8e40bb80b1442ecd2b0dfea00f5 Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Wed, 25 Mar 2026 09:28:50 -0600 Subject: [PATCH 2/5] docs: move envs/exit codes to their own help sections --- messages/agent.test.results.md | 9 --------- messages/agent.test.resume.md | 9 --------- messages/agent.test.run-eval.md | 9 --------- messages/agent.test.run.md | 9 --------- messages/agent.validate.authoring-bundle.md | 11 ----------- src/commands/agent/test/results.ts | 14 ++++++++++++-- src/commands/agent/test/resume.ts | 15 +++++++++++++-- src/commands/agent/test/run-eval.ts | 15 +++++++++++++-- src/commands/agent/test/run.ts | 15 +++++++++++++-- src/commands/agent/validate/authoring-bundle.ts | 17 +++++++++++++++-- 10 files changed, 66 insertions(+), 57 deletions(-) diff --git a/messages/agent.test.results.md b/messages/agent.test.results.md index 68008c6c..d3b1d0b4 100644 --- a/messages/agent.test.results.md +++ b/messages/agent.test.results.md @@ -8,15 +8,6 @@ This command requires a job ID, which the original "agent test run" command disp By default, this command outputs test results in human-readable tables for each test case. The tables show whether the test case passed, the expected and actual values, the test score, how long the test took, and more. Use the --result-format to display the test results in JSON or Junit format. Use the --output-dir flag to write the results to a file rather than to the terminal. -ERROR CODES - - Succeeded (0) Results retrieved successfully. Test results (passed/failed) are in the output. - Failed (1) Command couldn't execute due to invalid job ID, API errors, network issues, or system errors. - -ENVIRONMENT VARIABLES - - SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. - # flags.job-id.summary Job ID of the completed agent test run. diff --git a/messages/agent.test.resume.md b/messages/agent.test.resume.md index ef0feff4..cf0cfcfc 100644 --- a/messages/agent.test.resume.md +++ b/messages/agent.test.resume.md @@ -10,15 +10,6 @@ Use the --wait flag to specify the number of minutes for this command to wait fo By default, this command outputs test results in human-readable tables for each test case. The tables show whether the test case passed, the expected and actual values, the test score, how long the test took, and more. Use the --result-format to display the test results in JSON or Junit format. Use the --output-dir flag to write the results to a file rather than to the terminal. -ERROR CODES - - Succeeded (0) Test completed successfully. Test results (passed/failed) are in the JSON output. - Failed (1) Command couldn't execute due to invalid job ID, API errors, network issues, or system errors. Exit code 1 also indicates tests encountered execution errors. - -ENVIRONMENT VARIABLES - - SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. - # flags.job-id.summary Job ID of the original agent test run. diff --git a/messages/agent.test.run-eval.md b/messages/agent.test.run-eval.md index e1d6df12..304ab4bd 100644 --- a/messages/agent.test.run-eval.md +++ b/messages/agent.test.run-eval.md @@ -12,15 +12,6 @@ When you provide a JSON payload, it's sent directly to the API with optional nor Supports 8+ evaluator types, including topic routing assertions, action invocation checks, string/numeric assertions, semantic similarity scoring, and LLM-based quality ratings. -ERROR CODES - - Succeeded (0) Tests completed successfully. Test results (passed/failed) are in the JSON output. - Failed (1) Execution error occurred. Tests couldn't run due to API errors, network issues, invalid parameters, or system errors. - -ENVIRONMENT VARIABLES - - SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. - # flags.spec.summary Path to test spec file (YAML or JSON). Supports reading from stdin when piping content. diff --git a/messages/agent.test.run.md b/messages/agent.test.run.md index 4cfb2c15..09adc433 100644 --- a/messages/agent.test.run.md +++ b/messages/agent.test.run.md @@ -10,15 +10,6 @@ By default, this command starts the agent test in your org, but it doesn't wait By default, this command outputs test results in human-readable tables for each test case, if the test completes in time. The tables show whether the test case passed, the expected and actual values, the test score, how long the test took, and more. Use the --result-format to display the test results in JSON or Junit format. Use the --output-dir flag to write the results to a file rather than to the terminal. -ERROR CODES - - Succeeded (0) Test started successfully (without --wait), or test completed successfully (with --wait). - Failed (1) Command couldn't execute due to API errors, network issues, invalid test name, or system errors. When using --wait, exit code 1 also indicates tests encountered execution errors. - -ENVIRONMENT VARIABLES - - SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. - # flags.api-name.summary API name of the agent test to run; corresponds to the name of the AiEvaluationDefinition metadata component that implements the agent test. diff --git a/messages/agent.validate.authoring-bundle.md b/messages/agent.validate.authoring-bundle.md index 2c84291b..0a1ac99c 100644 --- a/messages/agent.validate.authoring-bundle.md +++ b/messages/agent.validate.authoring-bundle.md @@ -10,17 +10,6 @@ This command validates that the Agent Script file in the authoring bundle compil This command uses the API name of the authoring bundle. If you don't provide an API name with the --api-name flag, the command searches the current DX project and outputs a list of authoring bundles that it found for you to choose from. -ERROR CODES - - Succeeded (0) Agent Script file compiled successfully without errors. - Failed (1) Compilation errors found in the Agent Script file. - NotFound (2) Validation/compilation API returned HTTP 404. The API endpoint may not be available in your org or region. - ServerError (3) Validation/compilation API returned HTTP 500. A server error occurred during compilation. - -ENVIRONMENT VARIABLES - - SF_TARGET_ORG Username or alias of your default org. Overrides the target-org configuration variable. - # examples - Validate an authoring bundle by being prompted for its API name; use your default org: diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index 0dfbffd5..a72c83b2 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages } from '@salesforce/core'; import { AgentTester, AgentTestResultsResponse } from '@salesforce/agents'; import { resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js'; import { handleTestResults } from '../../../handleTestResults.js'; @@ -30,6 +30,16 @@ export default class AgentTestResults extends SfCommand public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Results retrieved successfully. Test results (passed/failed) are in the output.', + 'Failed (1)': "Command couldn't execute due to invalid job ID, API errors, network issues, or system errors.", + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index 7580ac49..e61a5bb9 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages } from '@salesforce/core'; import { AgentTester } from '@salesforce/agents'; import { AgentTestCache } from '../../../agentTestCache.js'; import { TestStages } from '../../../testStages.js'; @@ -30,6 +30,17 @@ export default class AgentTestResume extends SfCommand { public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Test completed successfully. Test results (passed/failed) are in the JSON output.', + 'Failed (1)': + "Command couldn't execute due to invalid job ID, API errors, network issues, or system errors. Exit code 1 also indicates tests encountered execution errors.", + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), diff --git a/src/commands/agent/test/run-eval.ts b/src/commands/agent/test/run-eval.ts index 3da15d75..0603425b 100644 --- a/src/commands/agent/test/run-eval.ts +++ b/src/commands/agent/test/run-eval.ts @@ -15,8 +15,8 @@ */ import { readFile } from 'node:fs/promises'; -import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages, Org } from '@salesforce/core'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages, Org } from '@salesforce/core'; import { type EvalPayload, normalizePayload, splitIntoBatches } from '../../../evalNormalizer.js'; import { type EvalApiResponse, formatResults, type ResultFormat } from '../../../evalFormatter.js'; import { resultFormatFlag } from '../../../flags.js'; @@ -157,6 +157,17 @@ export default class AgentTestRunEval extends SfCommand { public static state = 'beta'; public static readonly hidden = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Tests completed successfully. Test results (passed/failed) are in the JSON output.', + 'Failed (1)': + "Execution error occurred. Tests couldn't run due to API errors, network issues, invalid parameters, or system errors.", + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 11f24d83..9c9175a7 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages, SfError } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; import { AgentTester, AgentTestStartResponse } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; import { CLIError } from '@oclif/core/errors'; @@ -62,6 +62,17 @@ export default class AgentTestRun extends SfCommand { public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Test started successfully (without --wait), or test completed successfully (with --wait).', + 'Failed (1)': + "Command couldn't execute due to API errors, network issues, invalid test name, or system errors. When using --wait, exit code 1 also indicates tests encountered execution errors.", + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), diff --git a/src/commands/agent/validate/authoring-bundle.ts b/src/commands/agent/validate/authoring-bundle.ts index b179f48b..c5347db8 100644 --- a/src/commands/agent/validate/authoring-bundle.ts +++ b/src/commands/agent/validate/authoring-bundle.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages } from '@salesforce/core'; import { MultiStageOutput } from '@oclif/multi-stage-output'; import { Agent } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; @@ -35,6 +35,19 @@ export default class AgentValidateAuthoringBundle extends SfCommand Date: Wed, 25 Mar 2026 12:00:58 -0600 Subject: [PATCH 3/5] test: fix NUTs --- test/nuts/agent.test.run-eval.nut.ts | 111 ++++++++++++--------------- 1 file changed, 50 insertions(+), 61 deletions(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index ad5f83ad..dae9f307 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -39,107 +39,98 @@ describe('agent test run-eval', function () { describe('run-eval with JSON file', () => { it('should run evaluation with JSON payload file', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail (business logic), unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; - - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); - expect(output?.result.tests.length).to.be.greaterThan(0); - expect(output?.result.summary).to.be.ok; - expect(output?.result.summary.passed).to.be.a('number'); - expect(output?.result.summary.failed).to.be.a('number'); - expect(output?.result.summary.scored).to.be.a('number'); - expect(output?.result.summary.errors).to.be.a('number'); - // Verify no execution errors (only test failures are acceptable) - expect(output?.result.summary.errors).to.equal(0); + const result = execCmd(command, { ensureExitCode: 0 }); + + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); + expect(result.jsonOutput?.result.summary).to.be.ok; + expect(result.jsonOutput?.result.summary.passed).to.be.a('number'); + expect(result.jsonOutput?.result.summary.failed).to.be.a('number'); + expect(result.jsonOutput?.result.summary.scored).to.be.a('number'); + expect(result.jsonOutput?.result.summary.errors).to.be.a('number'); }); it('should run evaluation with normalized payload', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail, unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result.tests[0]).to.be.ok; - expect(output?.result.tests[0].id).to.equal('test-topic-routing'); - expect(output?.result.tests[0].status).to.be.oneOf(['passed', 'failed']); - expect(output?.result.tests[0].evaluations).to.be.an('array'); + expect(result.jsonOutput?.result.tests[0]).to.be.ok; + expect(result.jsonOutput?.result.tests[0].id).to.equal('test-topic-routing'); + expect(result.jsonOutput?.result.tests[0].status).to.be.oneOf(['passed', 'failed']); + expect(result.jsonOutput?.result.tests[0].evaluations).to.be.an('array'); }); }); describe('run-eval with YAML file', () => { it('should run evaluation with YAML test spec file', async () => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail, unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); - expect(output?.result.tests.length).to.be.greaterThan(0); - expect(output?.result.summary).to.be.ok; + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); + expect(result.jsonOutput?.result.summary).to.be.ok; }); it('should auto-infer agent name from YAML subjectName', async () => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail, unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); // Should succeed without explicit --api-name flag - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); }); it('should handle YAML spec with contextVariables', async () => { const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail, unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); - expect(output?.result.tests.length).to.be.greaterThan(0); - expect(output?.result.summary).to.be.ok; + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); + expect(result.jsonOutput?.result.summary).to.be.ok; }); }); describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail, unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); }); it('should use custom batch size', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail, unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); }); it('should support different result formats', async () => { - // Test human format (default) - exit code 0 even if tests fail + // Test human format (default) const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; - const humanOutput = execCmd(humanCommand, { ensureExitCode: 0 }).shellOutput.stdout; + const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); - expect(humanOutput).to.be.ok; - expect(humanOutput).to.be.a('string'); + expect(humanResult.shellOutput.stdout).to.be.ok; + expect(humanResult.shellOutput.stdout).to.be.a('string'); - // Test tap format - exit code 0 even if tests fail + // Test tap format const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; - const tapOutput = execCmd(tapCommand, { ensureExitCode: 0 }).shellOutput.stdout; + const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); - expect(tapOutput).to.include('TAP version'); + expect(tapResult.shellOutput.stdout).to.include('TAP version'); - // Test junit format - exit code 0 even if tests fail + // Test junit format const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; - const junitOutput = execCmd(junitCommand, { ensureExitCode: 0 }).shellOutput.stdout; + const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); - expect(junitOutput).to.include(' { it('should include test summaries with correct structure', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail, unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result.tests).to.be.an('array'); - const firstTest = output?.result.tests[0]; + expect(result.jsonOutput?.result.tests).to.be.an('array'); + const firstTest = result.jsonOutput?.result.tests[0]; expect(firstTest).to.have.property('id'); expect(firstTest).to.have.property('status'); expect(firstTest).to.have.property('evaluations'); @@ -200,10 +190,9 @@ describe('agent test run-eval', function () { it('should include summary with all metrics', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Exit code should be 0 even if tests fail, unless there are execution errors - const output = execCmd(command, { ensureExitCode: 0 }).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - const summary = output?.result.summary; + const summary = result.jsonOutput?.result.summary; expect(summary).to.have.property('passed'); expect(summary).to.have.property('failed'); expect(summary).to.have.property('scored'); From 06d70a6ca140c04b0b344a0908d4fbf6e2bbdf1e Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Wed, 25 Mar 2026 12:26:10 -0600 Subject: [PATCH 4/5] test: fix NUTs --- test/nuts/agent.test.run-eval.nut.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index dae9f307..12aa7353 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -97,7 +97,7 @@ describe('agent test run-eval', function () { describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; - const result = execCmd(command, { ensureExitCode: 0 }); + const result = execCmd(command, { ensureExitCode: 1 }); expect(result.jsonOutput?.result).to.be.ok; expect(result.jsonOutput?.result.tests).to.be.an('array'); From 85b249d2dfca15a674cc5293754d6b7c0c17a0e9 Mon Sep 17 00:00:00 2001 From: Willie Ruemmele Date: Fri, 27 Mar 2026 09:58:08 -0600 Subject: [PATCH 5/5] fix: expand error codes to other commands --- messages/agent.activate.md | 8 +++ messages/agent.deactivate.md | 8 +++ messages/agent.preview.end.md | 24 ++++++-- messages/agent.preview.send.md | 20 +++++-- messages/agent.preview.start.md | 26 +++++--- src/commands/agent/activate.ts | 56 +++++++++++++++-- src/commands/agent/deactivate.ts | 54 +++++++++++++++-- src/commands/agent/preview/end.ts | 83 +++++++++++++++++++++----- src/commands/agent/preview/send.ts | 76 +++++++++++++++++++---- src/commands/agent/preview/sessions.ts | 6 +- src/commands/agent/preview/start.ts | 82 +++++++++++++++++++++++-- src/commands/agent/test/create.ts | 49 +++++++++++++-- src/commands/agent/test/list.ts | 29 ++++++++- src/commands/agent/test/results.ts | 32 +++++++++- src/commands/agent/test/resume.ts | 71 ++++++++++++++++++---- src/commands/agent/test/run-eval.ts | 40 +++++++++++-- src/commands/agent/test/run.ts | 45 +++++++++++--- src/previewSessionStore.ts | 6 -- 18 files changed, 615 insertions(+), 100 deletions(-) diff --git a/messages/agent.activate.md b/messages/agent.activate.md index d03814e8..49844bf7 100644 --- a/messages/agent.activate.md +++ b/messages/agent.activate.md @@ -31,3 +31,11 @@ Version number of the agent to activate; if not specified, the command provides # error.missingRequiredFlags Missing required flags: %s. + +# error.agentNotFound + +Agent '%s' not found in the org. Check that the API name is correct and that the agent exists. + +# error.activationFailed + +Failed to activate agent: %s diff --git a/messages/agent.deactivate.md b/messages/agent.deactivate.md index 06f39755..fccfe7cc 100644 --- a/messages/agent.deactivate.md +++ b/messages/agent.deactivate.md @@ -25,3 +25,11 @@ API name of the agent to deactivate; if not specified, the command provides a li # error.missingRequiredFlags Missing required flags: %s. + +# error.agentNotFound + +Agent '%s' not found in the org. Check that the API name is correct and that the agent exists. + +# error.deactivationFailed + +Failed to deactivate agent: %s diff --git a/messages/agent.preview.end.md b/messages/agent.preview.end.md index 09a4d2f5..fc75cfe6 100644 --- a/messages/agent.preview.end.md +++ b/messages/agent.preview.end.md @@ -4,9 +4,9 @@ End an existing programmatic agent preview session and get trace location. # description -You must have previously started a programmatic agent preview session with the "agent preview start" command to then use this command to end it. This command also displays the local directory where the session trace files are stored. +You must have previously started a programmatic agent preview session with the "agent preview start" command to then use this command to end it. This command also displays the local directory where the session trace files are stored. -The original "agent preview start" command outputs a session ID which you then use with the --session-id flag of this command to end the session. You don't have to specify the --session-id flag if an agent has only one active preview session. You must also use either the --authoring-bundle or --api-name flag to specify the API name of the authoring bundle or the published agent, respecitvely. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. +The original "agent preview start" command outputs a session ID which you then use with the --session-id flag of this command to end the session. You don't have to specify the --session-id flag if an agent has only one active preview session. You must also use either the --authoring-bundle or --api-name flag to specify the API name of the authoring bundle or the published agent, respecitvely. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. # flags.session-id.summary @@ -14,7 +14,7 @@ Session ID outputted by "agent preview start". Not required when the agent has e # flags.api-name.summary -API name of the activated published agent you want to preview. +API name of the activated published agent you want to preview. # flags.authoring-bundle.summary @@ -28,6 +28,18 @@ No agent preview session found. Run "sf agent preview start" to start a new agen Multiple preview sessions found for this agent. Use the --session-id flag to identify a specific session. Sessions: %s +# error.agentNotFound + +Agent '%s' not found. Check that the API name is correct and that the agent exists in your org or project. + +# error.sessionInvalid + +Preview session '%s' is invalid or has expired. + +# error.endFailed + +Failed to end preview session: %s + # output.tracesPath Session traces: %s @@ -36,12 +48,12 @@ Session traces: %s - End a preview session of a published agent by specifying its session ID and API name ; use the default org: - <%= config.bin %> <%= command.id %> --session-id --api-name My_Published_Agent + <%= config.bin %> <%= command.id %> --session-id --api-name My_Published_Agent - Similar to previous example, but don't specify a session ID; you get an error if the published agent has more than one active session. Use the org with alias "my-dev-org": - <%= config.bin %> <%= command.id %> --api-name My_Published_Agent --target-org my-dev-org + <%= config.bin %> <%= command.id %> --api-name My_Published_Agent --target-org my-dev-org - End a preview session of an agent using its authoring bundle API name; you get an error if the agent has more than one active session. - <%= config.bin %> <%= command.id %> --authoring-bundle My_Local_Agent + <%= config.bin %> <%= command.id %> --authoring-bundle My_Local_Agent diff --git a/messages/agent.preview.send.md b/messages/agent.preview.send.md index 428ce765..5890e6cd 100644 --- a/messages/agent.preview.send.md +++ b/messages/agent.preview.send.md @@ -6,7 +6,7 @@ Send a message to an existing agent preview session. You must have previously started a programmatic agent preview session with the "agent preview start" command to then use this command to send the agent a message (utterance). This command then displays the agent's response. -The original "agent preview start" command outputs a session ID which you then use with the --session-id flag of this command to send a message. You don't have to specify the --session-id flag if an agent has only one active preview session. You must also use either the --authoring-bundle or --api-name flag to specify the API name of the authoring bundle or the published agent, respecitvely. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. +The original "agent preview start" command outputs a session ID which you then use with the --session-id flag of this command to send a message. You don't have to specify the --session-id flag if an agent has only one active preview session. You must also use either the --authoring-bundle or --api-name flag to specify the API name of the authoring bundle or the published agent, respecitvely. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. # flags.session-id.summary @@ -32,16 +32,28 @@ No agent preview session found. Run "sf agent preview start" to start a new agen Multiple preview sessions found for this agent. Use the --session-id flag to identify a specific session. Sessions: %s +# error.agentNotFound + +Agent '%s' not found. Check that the API name is correct and that the agent exists in your org or project. + +# error.sessionInvalid + +Preview session '%s' is invalid or has expired. Start a new session with "sf agent preview start". + +# error.sendFailed + +Failed to send message to preview session: %s + # examples - Send a message to an activated published agent using its API name and session ID; use the default org: - <%= config.bin %> <%= command.id %> --utterance "What can you help me with?" --api-name My_Published_Agent --session-id + <%= config.bin %> <%= command.id %> --utterance "What can you help me with?" --api-name My_Published_Agent --session-id - Similar to previous example, but don't specify a session ID; you get an error if the agent has more than one active session. Use the org with alias "my-dev-org": - <%= config.bin %> <%= command.id %> --utterance "What can you help me with?" --api-name My_Published_Agent --target-org my-dev-org + <%= config.bin %> <%= command.id %> --utterance "What can you help me with?" --api-name My_Published_Agent --target-org my-dev-org - Send a message to an agent using its authoring bundle API name; you get an error if the agent has more than one active session: - <%= config.bin %> <%= command.id %> --utterance "what can you help me with?" --authoring-bundle My_Local_Agent + <%= config.bin %> <%= command.id %> --utterance "what can you help me with?" --authoring-bundle My_Local_Agent diff --git a/messages/agent.preview.start.md b/messages/agent.preview.start.md index 3d840163..c326d99f 100644 --- a/messages/agent.preview.start.md +++ b/messages/agent.preview.start.md @@ -1,14 +1,14 @@ # summary -Start a programmatic agent preview session. +Start a programmatic agent preview session. # description -This command outputs a session ID that you then use with the "agent preview send" command to send an utterance to the agent. Use the "agent preview sessions" command to list all active sessions and the "agent preview end" command to end a specific session. +This command outputs a session ID that you then use with the "agent preview send" command to send an utterance to the agent. Use the "agent preview sessions" command to list all active sessions and the "agent preview end" command to end a specific session. -Identify the agent you want to start previewing with either the --authoring-bundle flag to specify a local authoring bundle's API name or --api-name to specify an activated published agent's API name. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. +Identify the agent you want to start previewing with either the --authoring-bundle flag to specify a local authoring bundle's API name or --api-name to specify an activated published agent's API name. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. -When starting a preview session using the authoring bundle, which contains the agent's Agent Script file, the preview uses mocked actions by default. Specify --use-live-actions for live mode, which uses the real Apex classes, flows, etc, in the org for the actions. +When starting a preview session using the authoring bundle, which contains the agent's Agent Script file, the preview uses mocked actions by default. Specify --use-live-actions for live mode, which uses the real Apex classes, flows, etc, in the org for the actions. # flags.api-name.summary @@ -26,16 +26,28 @@ Use real actions in the org; if not specified, preview uses AI to simulate (mock Session ID: %s +# error.agentNotFound + +Agent '%s' not found. Check that the API name is correct and that the agent exists in your org or project. + +# error.compilationFailed + +Agent Script compilation failed. See errors above for details. + +# error.previewStartFailed + +Failed to start preview session: %s + # examples - Start a programmatic agent preview session by specifying an authoring bundle; uses mocked actions by default. Use the org with alias "my-dev-org": - <%= config.bin %> <%= command.id %> --authoring-bundle My_Agent_Bundle --target-org my-dev-org + <%= config.bin %> <%= command.id %> --authoring-bundle My_Agent_Bundle --target-org my-dev-org - Similar to previous example but use live actions and the default org: - <%= config.bin %> <%= command.id %> --authoring-bundle My_Agent_Bundle --use-live-actions + <%= config.bin %> <%= command.id %> --authoring-bundle My_Agent_Bundle --use-live-actions - Start a preview session with an activated published agent: - <%= config.bin %> <%= command.id %> --api-name My_Published_Agent + <%= config.bin %> <%= command.id %> --api-name My_Published_Agent diff --git a/src/commands/agent/activate.ts b/src/commands/agent/activate.ts index 3d1b7035..6570b643 100644 --- a/src/commands/agent/activate.ts +++ b/src/commands/agent/activate.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError, Lifecycle, EnvironmentVariable } from '@salesforce/core'; import { getAgentForActivation, getVersionForActivation } from '../../agentActivation.js'; export type AgentActivateResult = { success: boolean; version: number }; @@ -28,6 +28,17 @@ export default class AgentActivate extends SfCommand { public static readonly examples = messages.getMessages('examples'); public static readonly enableJsonFlag = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Agent activated successfully.', + 'NotFound (2)': 'Agent not found in the org.', + 'ActivationFailed (4)': 'Failed to activate the agent due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -47,15 +58,52 @@ export default class AgentActivate extends SfCommand { if (!apiNameFlag && this.jsonEnabled()) { throw messages.createError('error.missingRequiredFlags', ['api-name']); } - const agent = await getAgentForActivation({ targetOrg, status: 'Active', apiNameFlag }); + + // Get agent with error tracking + let agent; + try { + agent = await getAgentForActivation({ targetOrg, status: 'Active', apiNameFlag }); + } catch (error) { + const wrapped = SfError.wrap(error); + if (wrapped.message.toLowerCase().includes('not found') || wrapped.message.toLowerCase().includes('no agent')) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_activate_agent_not_found' }); + throw new SfError( + messages.getMessage('error.agentNotFound', [apiNameFlag ?? 'unknown']), + 'AgentNotFound', + [], + 2, + wrapped + ); + } + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_activate_get_agent_failed' }); + throw wrapped; + } + const { version, warning } = await getVersionForActivation({ agent, status: 'Active', versionFlag: flags.version, jsonEnabled: this.jsonEnabled(), }); - const result = await agent.activate(version); + + // Activate with error tracking + let result; + try { + result = await agent.activate(version); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_activate_failed' }); + throw new SfError( + messages.getMessage('error.activationFailed', [wrapped.message]), + 'ActivationFailed', + [wrapped.message], + 4, + wrapped + ); + } + const metadata = await agent.getBotMetadata(); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_activate_success' }); this.log(`${metadata.DeveloperName} v${result.VersionNumber} activated.`); if (warning) { diff --git a/src/commands/agent/deactivate.ts b/src/commands/agent/deactivate.ts index c39019d8..d7ff8110 100644 --- a/src/commands/agent/deactivate.ts +++ b/src/commands/agent/deactivate.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError, Lifecycle, EnvironmentVariable } from '@salesforce/core'; import { getAgentForActivation } from '../../agentActivation.js'; import { AgentActivateResult } from './activate.js'; @@ -27,6 +27,17 @@ export default class AgentDeactivate extends SfCommand { public static readonly examples = messages.getMessages('examples'); public static enableJsonFlag = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Agent deactivated successfully.', + 'NotFound (2)': 'Agent not found in the org.', + 'DeactivationFailed (4)': 'Failed to deactivate the agent due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -46,9 +57,44 @@ export default class AgentDeactivate extends SfCommand { throw messages.createError('error.missingRequiredFlags', ['api-name']); } - const agent = await getAgentForActivation({ targetOrg, status: 'Inactive', apiNameFlag }); - const result = await agent.deactivate(); + // Get agent with error tracking + let agent; + try { + agent = await getAgentForActivation({ targetOrg, status: 'Inactive', apiNameFlag }); + } catch (error) { + const wrapped = SfError.wrap(error); + if (wrapped.message.toLowerCase().includes('not found') || wrapped.message.toLowerCase().includes('no agent')) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_deactivate_agent_not_found' }); + throw new SfError( + messages.getMessage('error.agentNotFound', [apiNameFlag ?? 'unknown']), + 'AgentNotFound', + [], + 2, + wrapped + ); + } + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_deactivate_get_agent_failed' }); + throw wrapped; + } + + // Deactivate with error tracking + let result; + try { + result = await agent.deactivate(); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_deactivate_failed' }); + throw new SfError( + messages.getMessage('error.deactivationFailed', [wrapped.message]), + 'DeactivationFailed', + [wrapped.message], + 4, + wrapped + ); + } + const metadata = await agent.getBotMetadata(); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_deactivate_success' }); this.log(`${metadata.DeveloperName} v${result.VersionNumber} deactivated.`); return { success: true, version: result.VersionNumber }; diff --git a/src/commands/agent/preview/end.ts b/src/commands/agent/preview/end.ts index 6ff970f5..4e801002 100644 --- a/src/commands/agent/preview/end.ts +++ b/src/commands/agent/preview/end.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages, SfError } from '@salesforce/core'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError, Lifecycle, EnvironmentVariable } from '@salesforce/core'; import { Agent, ProductionAgent, ScriptAgent } from '@salesforce/agents'; import { getCachedSessionIds, removeCache, validatePreviewSession } from '../../../previewSessionStore.js'; @@ -33,6 +33,18 @@ export default class AgentPreviewEnd extends SfCommand { public static readonly examples = messages.getMessages('examples'); public static readonly requiresProject = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Preview session ended successfully and traces saved.', + 'NotFound (2)': 'Agent not found, or no preview session exists for this agent.', + 'PreviewEndFailed (4)': 'Failed to end the preview session.', + 'SessionAmbiguous (5)': 'Multiple preview sessions found; specify --session-id to choose one.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -53,38 +65,81 @@ export default class AgentPreviewEnd extends SfCommand { public async run(): Promise { const { flags } = await this.parse(AgentPreviewEnd); - const conn = flags['target-org'].getConnection(flags['api-version']); - const agent = flags['authoring-bundle'] - ? await Agent.init({ connection: conn, project: this.project!, aabName: flags['authoring-bundle'] }) - : await Agent.init({ connection: conn, project: this.project!, apiNameOrId: flags['api-name']! }); + const agentIdentifier = flags['authoring-bundle'] ?? flags['api-name']!; + // Initialize agent with error tracking + let agent; + try { + agent = flags['authoring-bundle'] + ? await Agent.init({ connection: conn, project: this.project!, aabName: flags['authoring-bundle'] }) + : await Agent.init({ connection: conn, project: this.project!, apiNameOrId: flags['api-name']! }); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_agent_not_found' }); + throw new SfError(messages.getMessage('error.agentNotFound', [agentIdentifier]), 'AgentNotFound', [], 2, wrapped); + } + + // Get or validate session ID let sessionId = flags['session-id']; if (sessionId === undefined) { const cached = await getCachedSessionIds(this.project!, agent); if (cached.length === 0) { - throw new SfError(messages.getMessage('error.noSession'), 'PreviewSessionNotFound'); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_no_session' }); + throw new SfError(messages.getMessage('error.noSession'), 'PreviewSessionNotFound', [], 2); } if (cached.length > 1) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_multiple_sessions' }); throw new SfError( messages.getMessage('error.multipleSessions', [cached.join(', ')]), - 'PreviewSessionAmbiguous' + 'PreviewSessionAmbiguous', + [], + 5 ); } sessionId = cached[0]; } + agent.setSessionId(sessionId); - await validatePreviewSession(agent); - const tracesPath = await agent.getHistoryDir(); + // Validate session + try { + await validatePreviewSession(agent); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_session_invalid' }); + throw new SfError( + messages.getMessage('error.sessionInvalid', [sessionId]), + 'PreviewSessionInvalid', + [], + 2, + wrapped + ); + } + const tracesPath = await agent.getHistoryDir(); await removeCache(agent); - if (agent instanceof ScriptAgent) { - await agent.preview.end(); - } else if (agent instanceof ProductionAgent) { - await agent.preview.end('UserRequest'); + // End preview with error tracking + try { + if (agent instanceof ScriptAgent) { + await agent.preview.end(); + } else if (agent instanceof ProductionAgent) { + await agent.preview.end('UserRequest'); + } + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_failed' }); + throw new SfError( + messages.getMessage('error.endFailed', [wrapped.message]), + 'PreviewEndFailed', + [wrapped.message], + 4, + wrapped + ); } + + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_success' }); const result = { sessionId, tracesPath }; this.log(messages.getMessage('output.tracesPath', [tracesPath])); return result; diff --git a/src/commands/agent/preview/send.ts b/src/commands/agent/preview/send.ts index 36a9c67a..92dac6da 100644 --- a/src/commands/agent/preview/send.ts +++ b/src/commands/agent/preview/send.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages, SfError } from '@salesforce/core'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError, Lifecycle, EnvironmentVariable } from '@salesforce/core'; import { Agent } from '@salesforce/agents'; import { getCachedSessionIds, validatePreviewSession } from '../../../previewSessionStore.js'; @@ -32,6 +32,18 @@ export default class AgentPreviewSend extends SfCommand public static readonly examples = messages.getMessages('examples'); public static readonly requiresProject = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Message sent successfully and agent response received.', + 'NotFound (2)': 'Agent not found, or no preview session exists for this agent.', + 'PreviewSendFailed (4)': 'Failed to send message or receive response from the preview session.', + 'SessionAmbiguous (5)': 'Multiple preview sessions found; specify --session-id to choose one.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -57,31 +69,75 @@ export default class AgentPreviewSend extends SfCommand public async run(): Promise { const { flags } = await this.parse(AgentPreviewSend); - const conn = flags['target-org'].getConnection(flags['api-version']); + const agentIdentifier = flags['authoring-bundle'] ?? flags['api-name']!; - const agent = flags['authoring-bundle'] - ? await Agent.init({ connection: conn, project: this.project!, aabName: flags['authoring-bundle'] }) - : await Agent.init({ connection: conn, project: this.project!, apiNameOrId: flags['api-name']! }); + // Initialize agent with error tracking + let agent; + try { + agent = flags['authoring-bundle'] + ? await Agent.init({ connection: conn, project: this.project!, aabName: flags['authoring-bundle'] }) + : await Agent.init({ connection: conn, project: this.project!, apiNameOrId: flags['api-name']! }); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_agent_not_found' }); + throw new SfError(messages.getMessage('error.agentNotFound', [agentIdentifier]), 'AgentNotFound', [], 2, wrapped); + } + // Get or validate session ID let sessionId = flags['session-id']; if (sessionId === undefined) { const cached = await getCachedSessionIds(this.project!, agent); if (cached.length === 0) { - throw new SfError(messages.getMessage('error.noSession'), 'PreviewSessionNotFound'); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_no_session' }); + throw new SfError(messages.getMessage('error.noSession'), 'PreviewSessionNotFound', [], 2); } if (cached.length > 1) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_multiple_sessions' }); throw new SfError( messages.getMessage('error.multipleSessions', [cached.join(', ')]), - 'PreviewSessionAmbiguous' + 'PreviewSessionAmbiguous', + [], + 5 ); } sessionId = cached[0]; } + agent.setSessionId(sessionId); - await validatePreviewSession(agent); - const response = await agent.preview.send(flags.utterance); + // Validate session + try { + await validatePreviewSession(agent); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_session_invalid' }); + throw new SfError( + messages.getMessage('error.sessionInvalid', [sessionId]), + 'PreviewSessionInvalid', + [], + 2, + wrapped + ); + } + + // Send message with error tracking + let response; + try { + response = await agent.preview.send(flags.utterance); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_failed' }); + throw new SfError( + messages.getMessage('error.sendFailed', [wrapped.message]), + 'PreviewSendFailed', + [wrapped.message], + 4, + wrapped + ); + } + + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_success' }); this.log(response.messages[0].message); return { messages: response.messages ?? [] }; } diff --git a/src/commands/agent/preview/sessions.ts b/src/commands/agent/preview/sessions.ts index 95cb641f..ab15d221 100644 --- a/src/commands/agent/preview/sessions.ts +++ b/src/commands/agent/preview/sessions.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { SfCommand } from '@salesforce/sf-plugins-core'; +import { SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; import { listCachedSessions } from '../../../previewSessionStore.js'; @@ -29,6 +29,10 @@ export default class AgentPreviewSessions extends SfCommand { const entries = await listCachedSessions(this.project!); const rows: AgentPreviewSessionsResult = []; diff --git a/src/commands/agent/preview/start.ts b/src/commands/agent/preview/start.ts index 2e7d0bf4..738f40a7 100644 --- a/src/commands/agent/preview/start.ts +++ b/src/commands/agent/preview/start.ts @@ -14,10 +14,11 @@ * limitations under the License. */ -import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; -import { Lifecycle, Messages } from '@salesforce/core'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Lifecycle, Messages, SfError, EnvironmentVariable } from '@salesforce/core'; import { Agent, ProductionAgent, ScriptAgent } from '@salesforce/agents'; import { createCache } from '../../../previewSessionStore.js'; +import { COMPILATION_API_EXIT_CODES } from '../../../common.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.preview.start'); @@ -32,6 +33,20 @@ export default class AgentPreviewStart extends SfCommand { public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Test created and deployed successfully.', + 'Failed (1)': 'Test validation errors or metadata format issues.', + 'NotFound (2)': 'Test spec file not found or org connection failed.', + 'DeploymentFailed (4)': 'Deployment failed due to API or network errors.', + }); + public static readonly flags = { ...makeFlags(FLAGGABLE_PROMPTS), 'target-org': Flags.requiredOrg(), @@ -163,10 +175,35 @@ export default class AgentTestCreate extends SfCommand { return Promise.resolve(); }); - const { path, contents } = await AgentTest.create(connection, apiName, spec, { - outputDir: join('force-app', 'main', 'default', 'aiEvaluationDefinitions'), - preview: flags.preview, - }); + let path; + let contents; + try { + const result = await AgentTest.create(connection, apiName, spec, { + outputDir: join('force-app', 'main', 'default', 'aiEvaluationDefinitions'), + preview: flags.preview, + }); + path = result.path; + contents = result.contents; + } catch (error) { + const wrapped = SfError.wrap(error); + + // Check for file not found errors + if ( + wrapped.message.toLowerCase().includes('not found') || + wrapped.message.toLowerCase().includes('enoent') || + wrapped.code === 'ENOENT' + ) { + throw new SfError(`Test spec file not found: ${spec}`, 'SpecFileNotFound', [], 2, wrapped); + } + + // Check for deployment failures (API/network) + if (wrapped.message.toLowerCase().includes('deploy') || wrapped.message.toLowerCase().includes('api')) { + throw new SfError(`Deployment failed: ${wrapped.message}`, 'DeploymentFailed', [wrapped.message], 4, wrapped); + } + + // Other errors (validation, format issues) use exit 1 + throw wrapped; + } if (flags.preview) { this.mso?.skipTo(AgentTestCreateLifecycleStages.Done); diff --git a/src/commands/agent/test/list.ts b/src/commands/agent/test/list.ts index 88fedfa1..b4382b7e 100644 --- a/src/commands/agent/test/list.ts +++ b/src/commands/agent/test/list.ts @@ -15,8 +15,8 @@ */ import { AgentTest, type AvailableDefinition } from '@salesforce/agents'; -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, EnvironmentVariable, SfError } from '@salesforce/core'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.list'); @@ -28,6 +28,16 @@ export default class AgentTestList extends SfCommand { public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Agent tests listed successfully.', + 'Failed (4)': 'Failed to retrieve agent tests due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -36,7 +46,20 @@ export default class AgentTestList extends SfCommand { public async run(): Promise { const { flags } = await this.parse(AgentTestList); - const results = await AgentTest.list(flags['target-org'].getConnection(flags['api-version'])); + let results; + try { + results = await AgentTest.list(flags['target-org'].getConnection(flags['api-version'])); + } catch (error) { + const wrapped = SfError.wrap(error); + throw new SfError( + `Failed to retrieve agent tests: ${wrapped.message}`, + 'ListRetrievalFailed', + [wrapped.message], + 4, + wrapped + ); + } + this.table({ data: results, columns: [ diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index a72c83b2..69cfe77e 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -15,7 +15,7 @@ */ import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; -import { EnvironmentVariable, Messages } from '@salesforce/core'; +import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; import { AgentTester, AgentTestResultsResponse } from '@salesforce/agents'; import { resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js'; import { handleTestResults } from '../../../handleTestResults.js'; @@ -37,7 +37,8 @@ export default class AgentTestResults extends SfCommand public static readonly errorCodes = toHelpSection('ERROR CODES', { 'Succeeded (0)': 'Results retrieved successfully. Test results (passed/failed) are in the output.', - 'Failed (1)': "Command couldn't execute due to invalid job ID, API errors, network issues, or system errors.", + 'NotFound (2)': 'Job ID not found or invalid.', + 'Failed (4)': 'Failed to retrieve results due to API or network errors.', }); public static readonly flags = { @@ -57,7 +58,32 @@ export default class AgentTestResults extends SfCommand const { flags } = await this.parse(AgentTestResults); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const response = await agentTester.results(flags['job-id']); + + let response; + try { + response = await agentTester.results(flags['job-id']); + } catch (error) { + const wrapped = SfError.wrap(error); + + // Check for job not found errors + if ( + wrapped.message.toLowerCase().includes('not found') || + wrapped.message.toLowerCase().includes('invalid') || + wrapped.code === 'ENOENT' + ) { + throw new SfError(`Job ID '${flags['job-id']}' not found or invalid.`, 'JobNotFound', [], 2, wrapped); + } + + // API/network failures + throw new SfError( + `Failed to retrieve results: ${wrapped.message}`, + 'ResultsRetrievalFailed', + [wrapped.message], + 4, + wrapped + ); + } + await handleTestResults({ id: flags['job-id'], format: flags['result-format'], diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index e61a5bb9..bca01e77 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -15,8 +15,9 @@ */ import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; -import { EnvironmentVariable, Messages } from '@salesforce/core'; +import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; import { AgentTester } from '@salesforce/agents'; +import { CLIError } from '@oclif/core/errors'; import { AgentTestCache } from '../../../agentTestCache.js'; import { TestStages } from '../../../testStages.js'; import { AgentTestRunResult, resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js'; @@ -36,9 +37,10 @@ export default class AgentTestResume extends SfCommand { ); public static readonly errorCodes = toHelpSection('ERROR CODES', { - 'Succeeded (0)': 'Test completed successfully. Test results (passed/failed) are in the JSON output.', - 'Failed (1)': - "Command couldn't execute due to invalid job ID, API errors, network issues, or system errors. Exit code 1 also indicates tests encountered execution errors.", + 'Succeeded (0)': 'Test completed successfully (with test results in the output).', + 'Failed (1)': 'Tests encountered execution errors (test cases with ERROR status).', + 'NotFound (2)': 'Job ID not found or invalid.', + 'OperationFailed (4)': 'Failed to poll test due to API or network errors.', }); public static readonly flags = { @@ -66,26 +68,65 @@ export default class AgentTestResume extends SfCommand { verbose: verboseFlag, }; + private mso: TestStages | undefined; + public async run(): Promise { const { flags } = await this.parse(AgentTestResume); const agentTestCache = await AgentTestCache.create(); - const { name, runId, outputDir, resultFormat } = agentTestCache.useIdOrMostRecent( - flags['job-id'], - flags['use-most-recent'] - ); + let name; + let runId; + let outputDir; + let resultFormat; + + try { + const cacheEntry = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); + name = cacheEntry.name; + runId = cacheEntry.runId; + outputDir = cacheEntry.outputDir; + resultFormat = cacheEntry.resultFormat; + } catch (e) { + const wrapped = SfError.wrap(e); + + // Check for job not found + if ( + wrapped.message.toLowerCase().includes('not found') || + wrapped.message.toLowerCase().includes('no test') || + wrapped.message.toLowerCase().includes('invalid') + ) { + throw new SfError(`Job ID '${flags['job-id'] ?? 'most recent'}' not found.`, 'JobNotFound', [], 2, wrapped); + } + + throw wrapped; + } - const mso = new TestStages({ + this.mso = new TestStages({ title: `Agent Test Run: ${name ?? runId}`, jsonEnabled: this.jsonEnabled(), }); - mso.start({ id: runId }); + this.mso.start({ id: runId }); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const { completed, response } = await mso.poll(agentTester, runId, flags.wait); + let completed; + let response; + try { + const pollResult = await this.mso.poll(agentTester, runId, flags.wait); + completed = pollResult.completed; + response = pollResult.response; + } catch (error) { + const wrapped = SfError.wrap(error); + throw new SfError( + `Failed to poll test results: ${wrapped.message}`, + 'TestPollFailed', + [wrapped.message], + 4, + wrapped + ); + } + if (completed) await agentTestCache.removeCacheEntry(runId); - mso.stop(); + this.mso.stop(); await handleTestResults({ id: runId, @@ -102,6 +143,12 @@ export default class AgentTestResume extends SfCommand { process.exitCode = 1; } + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion return { ...response!, runId, status: 'COMPLETED' }; } + + protected catch(error: Error | SfError | CLIError): Promise { + this.mso?.error(); + return super.catch(error); + } } diff --git a/src/commands/agent/test/run-eval.ts b/src/commands/agent/test/run-eval.ts index 0603425b..d2569526 100644 --- a/src/commands/agent/test/run-eval.ts +++ b/src/commands/agent/test/run-eval.ts @@ -16,7 +16,7 @@ import { readFile } from 'node:fs/promises'; import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; -import { EnvironmentVariable, Messages, Org } from '@salesforce/core'; +import { EnvironmentVariable, Messages, Org, SfError } from '@salesforce/core'; import { type EvalPayload, normalizePayload, splitIntoBatches } from '../../../evalNormalizer.js'; import { type EvalApiResponse, formatResults, type ResultFormat } from '../../../evalFormatter.js'; import { resultFormatFlag } from '../../../flags.js'; @@ -164,8 +164,9 @@ export default class AgentTestRunEval extends SfCommand { public static readonly errorCodes = toHelpSection('ERROR CODES', { 'Succeeded (0)': 'Tests completed successfully. Test results (passed/failed) are in the JSON output.', - 'Failed (1)': - "Execution error occurred. Tests couldn't run due to API errors, network issues, invalid parameters, or system errors.", + 'Failed (1)': "Tests encountered execution errors (tests couldn't run properly).", + 'NotFound (2)': 'Agent not found, spec file not found, or invalid agent name.', + 'OperationFailed (4)': 'Failed to execute tests due to API or network errors.', }); public static readonly flags = { @@ -214,7 +215,12 @@ export default class AgentTestRunEval extends SfCommand { // If we got here, it's valid content } catch { // Not valid content, must be a file path - read it - rawContent = await readFile(flags.spec, 'utf-8'); + try { + rawContent = await readFile(flags.spec, 'utf-8'); + } catch (e) { + const wrapped = SfError.wrap(e); + throw new SfError(`Spec file not found: ${flags.spec}`, 'SpecFileNotFound', [], 2, wrapped); + } } // 2. Detect format and parse @@ -246,7 +252,17 @@ export default class AgentTestRunEval extends SfCommand { // 3. If --api-name (or auto-inferred from YAML), resolve IDs and inject if (agentApiName) { - const { agentId, versionId } = await resolveAgent(org, agentApiName); + let agentId; + let versionId; + try { + const resolved = await resolveAgent(org, agentApiName); + agentId = resolved.agentId; + versionId = resolved.versionId; + } catch (e) { + const wrapped = SfError.wrap(e); + throw new SfError(`Agent '${agentApiName}' not found.`, 'AgentNotFound', [], 2, wrapped); + } + for (const test of payload.tests) { for (const step of test.steps) { if (step.type === 'agent.create_session') { @@ -271,7 +287,19 @@ export default class AgentTestRunEval extends SfCommand { const batches = splitIntoBatches(payload.tests, batchSize); // 7. Execute batches - const allResults = await executeBatches(org, batches, (msg) => this.log(msg)); + let allResults; + try { + allResults = await executeBatches(org, batches, (msg) => this.log(msg)); + } catch (e) { + const wrapped = SfError.wrap(e); + throw new SfError( + `Failed to execute tests: ${wrapped.message}`, + 'TestExecutionFailed', + [wrapped.message], + 4, + wrapped + ); + } const mergedResponse: EvalApiResponse = { results: allResults as EvalApiResponse['results'] }; diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 9c9175a7..330223c6 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -69,8 +69,9 @@ export default class AgentTestRun extends SfCommand { public static readonly errorCodes = toHelpSection('ERROR CODES', { 'Succeeded (0)': 'Test started successfully (without --wait), or test completed successfully (with --wait).', - 'Failed (1)': - "Command couldn't execute due to API errors, network issues, invalid test name, or system errors. When using --wait, exit code 1 also indicates tests encountered execution errors.", + 'Failed (1)': 'Tests encountered execution errors (test cases with ERROR status when using --wait).', + 'NotFound (2)': 'Test definition not found or invalid test name.', + 'OperationFailed (4)': 'Failed to start or poll test due to API or network errors.', }); public static readonly flags = { @@ -112,12 +113,23 @@ export default class AgentTestRun extends SfCommand { response = await agentTester.start(apiName); } catch (e) { const wrapped = SfError.wrap(e); - if (wrapped.message.includes('Invalid AiEvalDefinitionVersion identifier')) { - wrapped.actions = [ - `Try running "sf agent test list -o ${flags['target-org'].getUsername() ?? ''}" to see available options`, - ]; + + // Check for test definition not found + if ( + wrapped.message.includes('Invalid AiEvalDefinitionVersion identifier') || + wrapped.message.toLowerCase().includes('not found') + ) { + throw new SfError( + `Test definition '${apiName}' not found.`, + 'TestNotFound', + [`Try running "sf agent test list -o ${flags['target-org'].getUsername() ?? ''}" to see available options`], + 2, + wrapped + ); } - throw wrapped; + + // API/network failures + throw new SfError(`Failed to start test: ${wrapped.message}`, 'TestStartFailed', [wrapped.message], 4, wrapped); } this.mso.update({ id: response.runId }); @@ -126,7 +138,23 @@ export default class AgentTestRun extends SfCommand { await agentTestCache.createCacheEntry(response.runId, apiName, flags['output-dir'], flags['result-format']); if (flags.wait?.minutes) { - const { completed, response: detailsResponse } = await this.mso.poll(agentTester, response.runId, flags.wait); + let completed; + let detailsResponse; + try { + const pollResult = await this.mso.poll(agentTester, response.runId, flags.wait); + completed = pollResult.completed; + detailsResponse = pollResult.response; + } catch (error) { + const wrapped = SfError.wrap(error); + throw new SfError( + `Failed to poll test results: ${wrapped.message}`, + 'TestPollFailed', + [wrapped.message], + 4, + wrapped + ); + } + if (completed) await agentTestCache.removeCacheEntry(response.runId); this.mso.stop(); @@ -146,6 +174,7 @@ export default class AgentTestRun extends SfCommand { process.exitCode = 1; } + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion return { ...detailsResponse!, status: 'COMPLETED', runId: response.runId }; } else { this.mso.stop(); diff --git a/src/previewSessionStore.ts b/src/previewSessionStore.ts index a6593d84..ac90d843 100644 --- a/src/previewSessionStore.ts +++ b/src/previewSessionStore.ts @@ -33,10 +33,8 @@ export async function createCache( agent: ScriptAgent | ProductionAgent, options?: { displayName?: string } ): Promise { - /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ const historyDir = await agent.getHistoryDir(); const metaPath = join(historyDir, SESSION_META_FILE); - /* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ const meta: SessionMeta = { displayName: options?.displayName }; await writeFile(metaPath, JSON.stringify(meta), 'utf-8'); } @@ -47,10 +45,8 @@ export async function createCache( * Throws SfError if the session marker is not found. */ export async function validatePreviewSession(agent: ScriptAgent | ProductionAgent): Promise { - /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ const historyDir = await agent.getHistoryDir(); const metaPath = join(historyDir, SESSION_META_FILE); - /* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ try { await readFile(metaPath, 'utf-8'); } catch { @@ -66,10 +62,8 @@ export async function validatePreviewSession(agent: ScriptAgent | ProductionAgen * Call after ending the session. Caller must set sessionId on the agent before calling. */ export async function removeCache(agent: ScriptAgent | ProductionAgent): Promise { - /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ const historyDir = await agent.getHistoryDir(); const metaPath = join(historyDir, SESSION_META_FILE); - /* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ try { await unlink(metaPath); } catch {