diff --git a/messages/agent.activate.md b/messages/agent.activate.md index d03814e8..49844bf7 100644 --- a/messages/agent.activate.md +++ b/messages/agent.activate.md @@ -31,3 +31,11 @@ Version number of the agent to activate; if not specified, the command provides # error.missingRequiredFlags Missing required flags: %s. + +# error.agentNotFound + +Agent '%s' not found in the org. Check that the API name is correct and that the agent exists. + +# error.activationFailed + +Failed to activate agent: %s diff --git a/messages/agent.deactivate.md b/messages/agent.deactivate.md index 06f39755..fccfe7cc 100644 --- a/messages/agent.deactivate.md +++ b/messages/agent.deactivate.md @@ -25,3 +25,11 @@ API name of the agent to deactivate; if not specified, the command provides a li # error.missingRequiredFlags Missing required flags: %s. + +# error.agentNotFound + +Agent '%s' not found in the org. Check that the API name is correct and that the agent exists. + +# error.deactivationFailed + +Failed to deactivate agent: %s diff --git a/messages/agent.preview.end.md b/messages/agent.preview.end.md index 09a4d2f5..fc75cfe6 100644 --- a/messages/agent.preview.end.md +++ b/messages/agent.preview.end.md @@ -4,9 +4,9 @@ End an existing programmatic agent preview session and get trace location. # description -You must have previously started a programmatic agent preview session with the "agent preview start" command to then use this command to end it. This command also displays the local directory where the session trace files are stored. +You must have previously started a programmatic agent preview session with the "agent preview start" command to then use this command to end it. This command also displays the local directory where the session trace files are stored. -The original "agent preview start" command outputs a session ID which you then use with the --session-id flag of this command to end the session. You don't have to specify the --session-id flag if an agent has only one active preview session. You must also use either the --authoring-bundle or --api-name flag to specify the API name of the authoring bundle or the published agent, respecitvely. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. +The original "agent preview start" command outputs a session ID which you then use with the --session-id flag of this command to end the session. You don't have to specify the --session-id flag if an agent has only one active preview session. You must also use either the --authoring-bundle or --api-name flag to specify the API name of the authoring bundle or the published agent, respecitvely. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. # flags.session-id.summary @@ -14,7 +14,7 @@ Session ID outputted by "agent preview start". Not required when the agent has e # flags.api-name.summary -API name of the activated published agent you want to preview. +API name of the activated published agent you want to preview. # flags.authoring-bundle.summary @@ -28,6 +28,18 @@ No agent preview session found. Run "sf agent preview start" to start a new agen Multiple preview sessions found for this agent. Use the --session-id flag to identify a specific session. Sessions: %s +# error.agentNotFound + +Agent '%s' not found. Check that the API name is correct and that the agent exists in your org or project. + +# error.sessionInvalid + +Preview session '%s' is invalid or has expired. + +# error.endFailed + +Failed to end preview session: %s + # output.tracesPath Session traces: %s @@ -36,12 +48,12 @@ Session traces: %s - End a preview session of a published agent by specifying its session ID and API name ; use the default org: - <%= config.bin %> <%= command.id %> --session-id --api-name My_Published_Agent + <%= config.bin %> <%= command.id %> --session-id --api-name My_Published_Agent - Similar to previous example, but don't specify a session ID; you get an error if the published agent has more than one active session. Use the org with alias "my-dev-org": - <%= config.bin %> <%= command.id %> --api-name My_Published_Agent --target-org my-dev-org + <%= config.bin %> <%= command.id %> --api-name My_Published_Agent --target-org my-dev-org - End a preview session of an agent using its authoring bundle API name; you get an error if the agent has more than one active session. - <%= config.bin %> <%= command.id %> --authoring-bundle My_Local_Agent + <%= config.bin %> <%= command.id %> --authoring-bundle My_Local_Agent diff --git a/messages/agent.preview.send.md b/messages/agent.preview.send.md index 428ce765..5890e6cd 100644 --- a/messages/agent.preview.send.md +++ b/messages/agent.preview.send.md @@ -6,7 +6,7 @@ Send a message to an existing agent preview session. You must have previously started a programmatic agent preview session with the "agent preview start" command to then use this command to send the agent a message (utterance). This command then displays the agent's response. -The original "agent preview start" command outputs a session ID which you then use with the --session-id flag of this command to send a message. You don't have to specify the --session-id flag if an agent has only one active preview session. You must also use either the --authoring-bundle or --api-name flag to specify the API name of the authoring bundle or the published agent, respecitvely. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. +The original "agent preview start" command outputs a session ID which you then use with the --session-id flag of this command to send a message. You don't have to specify the --session-id flag if an agent has only one active preview session. You must also use either the --authoring-bundle or --api-name flag to specify the API name of the authoring bundle or the published agent, respecitvely. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. # flags.session-id.summary @@ -32,16 +32,28 @@ No agent preview session found. Run "sf agent preview start" to start a new agen Multiple preview sessions found for this agent. Use the --session-id flag to identify a specific session. Sessions: %s +# error.agentNotFound + +Agent '%s' not found. Check that the API name is correct and that the agent exists in your org or project. + +# error.sessionInvalid + +Preview session '%s' is invalid or has expired. Start a new session with "sf agent preview start". + +# error.sendFailed + +Failed to send message to preview session: %s + # examples - Send a message to an activated published agent using its API name and session ID; use the default org: - <%= config.bin %> <%= command.id %> --utterance "What can you help me with?" --api-name My_Published_Agent --session-id + <%= config.bin %> <%= command.id %> --utterance "What can you help me with?" --api-name My_Published_Agent --session-id - Similar to previous example, but don't specify a session ID; you get an error if the agent has more than one active session. Use the org with alias "my-dev-org": - <%= config.bin %> <%= command.id %> --utterance "What can you help me with?" --api-name My_Published_Agent --target-org my-dev-org + <%= config.bin %> <%= command.id %> --utterance "What can you help me with?" --api-name My_Published_Agent --target-org my-dev-org - Send a message to an agent using its authoring bundle API name; you get an error if the agent has more than one active session: - <%= config.bin %> <%= command.id %> --utterance "what can you help me with?" --authoring-bundle My_Local_Agent + <%= config.bin %> <%= command.id %> --utterance "what can you help me with?" --authoring-bundle My_Local_Agent diff --git a/messages/agent.preview.start.md b/messages/agent.preview.start.md index 3d840163..c326d99f 100644 --- a/messages/agent.preview.start.md +++ b/messages/agent.preview.start.md @@ -1,14 +1,14 @@ # summary -Start a programmatic agent preview session. +Start a programmatic agent preview session. # description -This command outputs a session ID that you then use with the "agent preview send" command to send an utterance to the agent. Use the "agent preview sessions" command to list all active sessions and the "agent preview end" command to end a specific session. +This command outputs a session ID that you then use with the "agent preview send" command to send an utterance to the agent. Use the "agent preview sessions" command to list all active sessions and the "agent preview end" command to end a specific session. -Identify the agent you want to start previewing with either the --authoring-bundle flag to specify a local authoring bundle's API name or --api-name to specify an activated published agent's API name. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. +Identify the agent you want to start previewing with either the --authoring-bundle flag to specify a local authoring bundle's API name or --api-name to specify an activated published agent's API name. To find either API name, navigate to your package directory in your DX project. The API name of an authoring bundle is the same as its directory name under the "aiAuthoringBundles" metadata directory. Similarly, the published agent's API name is the same as its directory name under the "Bots" metadata directory. -When starting a preview session using the authoring bundle, which contains the agent's Agent Script file, the preview uses mocked actions by default. Specify --use-live-actions for live mode, which uses the real Apex classes, flows, etc, in the org for the actions. +When starting a preview session using the authoring bundle, which contains the agent's Agent Script file, the preview uses mocked actions by default. Specify --use-live-actions for live mode, which uses the real Apex classes, flows, etc, in the org for the actions. # flags.api-name.summary @@ -26,16 +26,28 @@ Use real actions in the org; if not specified, preview uses AI to simulate (mock Session ID: %s +# error.agentNotFound + +Agent '%s' not found. Check that the API name is correct and that the agent exists in your org or project. + +# error.compilationFailed + +Agent Script compilation failed. See errors above for details. + +# error.previewStartFailed + +Failed to start preview session: %s + # examples - Start a programmatic agent preview session by specifying an authoring bundle; uses mocked actions by default. Use the org with alias "my-dev-org": - <%= config.bin %> <%= command.id %> --authoring-bundle My_Agent_Bundle --target-org my-dev-org + <%= config.bin %> <%= command.id %> --authoring-bundle My_Agent_Bundle --target-org my-dev-org - Similar to previous example but use live actions and the default org: - <%= config.bin %> <%= command.id %> --authoring-bundle My_Agent_Bundle --use-live-actions + <%= config.bin %> <%= command.id %> --authoring-bundle My_Agent_Bundle --use-live-actions - Start a preview session with an activated published agent: - <%= config.bin %> <%= command.id %> --api-name My_Published_Agent + <%= config.bin %> <%= command.id %> --api-name My_Published_Agent diff --git a/src/commands/agent/activate.ts b/src/commands/agent/activate.ts index 3d1b7035..6570b643 100644 --- a/src/commands/agent/activate.ts +++ b/src/commands/agent/activate.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError, Lifecycle, EnvironmentVariable } from '@salesforce/core'; import { getAgentForActivation, getVersionForActivation } from '../../agentActivation.js'; export type AgentActivateResult = { success: boolean; version: number }; @@ -28,6 +28,17 @@ export default class AgentActivate extends SfCommand { public static readonly examples = messages.getMessages('examples'); public static readonly enableJsonFlag = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Agent activated successfully.', + 'NotFound (2)': 'Agent not found in the org.', + 'ActivationFailed (4)': 'Failed to activate the agent due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -47,15 +58,52 @@ export default class AgentActivate extends SfCommand { if (!apiNameFlag && this.jsonEnabled()) { throw messages.createError('error.missingRequiredFlags', ['api-name']); } - const agent = await getAgentForActivation({ targetOrg, status: 'Active', apiNameFlag }); + + // Get agent with error tracking + let agent; + try { + agent = await getAgentForActivation({ targetOrg, status: 'Active', apiNameFlag }); + } catch (error) { + const wrapped = SfError.wrap(error); + if (wrapped.message.toLowerCase().includes('not found') || wrapped.message.toLowerCase().includes('no agent')) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_activate_agent_not_found' }); + throw new SfError( + messages.getMessage('error.agentNotFound', [apiNameFlag ?? 'unknown']), + 'AgentNotFound', + [], + 2, + wrapped + ); + } + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_activate_get_agent_failed' }); + throw wrapped; + } + const { version, warning } = await getVersionForActivation({ agent, status: 'Active', versionFlag: flags.version, jsonEnabled: this.jsonEnabled(), }); - const result = await agent.activate(version); + + // Activate with error tracking + let result; + try { + result = await agent.activate(version); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_activate_failed' }); + throw new SfError( + messages.getMessage('error.activationFailed', [wrapped.message]), + 'ActivationFailed', + [wrapped.message], + 4, + wrapped + ); + } + const metadata = await agent.getBotMetadata(); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_activate_success' }); this.log(`${metadata.DeveloperName} v${result.VersionNumber} activated.`); if (warning) { diff --git a/src/commands/agent/deactivate.ts b/src/commands/agent/deactivate.ts index c39019d8..d7ff8110 100644 --- a/src/commands/agent/deactivate.ts +++ b/src/commands/agent/deactivate.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError, Lifecycle, EnvironmentVariable } from '@salesforce/core'; import { getAgentForActivation } from '../../agentActivation.js'; import { AgentActivateResult } from './activate.js'; @@ -27,6 +27,17 @@ export default class AgentDeactivate extends SfCommand { public static readonly examples = messages.getMessages('examples'); public static enableJsonFlag = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Agent deactivated successfully.', + 'NotFound (2)': 'Agent not found in the org.', + 'DeactivationFailed (4)': 'Failed to deactivate the agent due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -46,9 +57,44 @@ export default class AgentDeactivate extends SfCommand { throw messages.createError('error.missingRequiredFlags', ['api-name']); } - const agent = await getAgentForActivation({ targetOrg, status: 'Inactive', apiNameFlag }); - const result = await agent.deactivate(); + // Get agent with error tracking + let agent; + try { + agent = await getAgentForActivation({ targetOrg, status: 'Inactive', apiNameFlag }); + } catch (error) { + const wrapped = SfError.wrap(error); + if (wrapped.message.toLowerCase().includes('not found') || wrapped.message.toLowerCase().includes('no agent')) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_deactivate_agent_not_found' }); + throw new SfError( + messages.getMessage('error.agentNotFound', [apiNameFlag ?? 'unknown']), + 'AgentNotFound', + [], + 2, + wrapped + ); + } + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_deactivate_get_agent_failed' }); + throw wrapped; + } + + // Deactivate with error tracking + let result; + try { + result = await agent.deactivate(); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_deactivate_failed' }); + throw new SfError( + messages.getMessage('error.deactivationFailed', [wrapped.message]), + 'DeactivationFailed', + [wrapped.message], + 4, + wrapped + ); + } + const metadata = await agent.getBotMetadata(); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_deactivate_success' }); this.log(`${metadata.DeveloperName} v${result.VersionNumber} deactivated.`); return { success: true, version: result.VersionNumber }; diff --git a/src/commands/agent/preview/end.ts b/src/commands/agent/preview/end.ts index 6ff970f5..4e801002 100644 --- a/src/commands/agent/preview/end.ts +++ b/src/commands/agent/preview/end.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages, SfError } from '@salesforce/core'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError, Lifecycle, EnvironmentVariable } from '@salesforce/core'; import { Agent, ProductionAgent, ScriptAgent } from '@salesforce/agents'; import { getCachedSessionIds, removeCache, validatePreviewSession } from '../../../previewSessionStore.js'; @@ -33,6 +33,18 @@ export default class AgentPreviewEnd extends SfCommand { public static readonly examples = messages.getMessages('examples'); public static readonly requiresProject = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Preview session ended successfully and traces saved.', + 'NotFound (2)': 'Agent not found, or no preview session exists for this agent.', + 'PreviewEndFailed (4)': 'Failed to end the preview session.', + 'SessionAmbiguous (5)': 'Multiple preview sessions found; specify --session-id to choose one.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -53,38 +65,81 @@ export default class AgentPreviewEnd extends SfCommand { public async run(): Promise { const { flags } = await this.parse(AgentPreviewEnd); - const conn = flags['target-org'].getConnection(flags['api-version']); - const agent = flags['authoring-bundle'] - ? await Agent.init({ connection: conn, project: this.project!, aabName: flags['authoring-bundle'] }) - : await Agent.init({ connection: conn, project: this.project!, apiNameOrId: flags['api-name']! }); + const agentIdentifier = flags['authoring-bundle'] ?? flags['api-name']!; + // Initialize agent with error tracking + let agent; + try { + agent = flags['authoring-bundle'] + ? await Agent.init({ connection: conn, project: this.project!, aabName: flags['authoring-bundle'] }) + : await Agent.init({ connection: conn, project: this.project!, apiNameOrId: flags['api-name']! }); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_agent_not_found' }); + throw new SfError(messages.getMessage('error.agentNotFound', [agentIdentifier]), 'AgentNotFound', [], 2, wrapped); + } + + // Get or validate session ID let sessionId = flags['session-id']; if (sessionId === undefined) { const cached = await getCachedSessionIds(this.project!, agent); if (cached.length === 0) { - throw new SfError(messages.getMessage('error.noSession'), 'PreviewSessionNotFound'); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_no_session' }); + throw new SfError(messages.getMessage('error.noSession'), 'PreviewSessionNotFound', [], 2); } if (cached.length > 1) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_multiple_sessions' }); throw new SfError( messages.getMessage('error.multipleSessions', [cached.join(', ')]), - 'PreviewSessionAmbiguous' + 'PreviewSessionAmbiguous', + [], + 5 ); } sessionId = cached[0]; } + agent.setSessionId(sessionId); - await validatePreviewSession(agent); - const tracesPath = await agent.getHistoryDir(); + // Validate session + try { + await validatePreviewSession(agent); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_session_invalid' }); + throw new SfError( + messages.getMessage('error.sessionInvalid', [sessionId]), + 'PreviewSessionInvalid', + [], + 2, + wrapped + ); + } + const tracesPath = await agent.getHistoryDir(); await removeCache(agent); - if (agent instanceof ScriptAgent) { - await agent.preview.end(); - } else if (agent instanceof ProductionAgent) { - await agent.preview.end('UserRequest'); + // End preview with error tracking + try { + if (agent instanceof ScriptAgent) { + await agent.preview.end(); + } else if (agent instanceof ProductionAgent) { + await agent.preview.end('UserRequest'); + } + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_failed' }); + throw new SfError( + messages.getMessage('error.endFailed', [wrapped.message]), + 'PreviewEndFailed', + [wrapped.message], + 4, + wrapped + ); } + + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_end_success' }); const result = { sessionId, tracesPath }; this.log(messages.getMessage('output.tracesPath', [tracesPath])); return result; diff --git a/src/commands/agent/preview/send.ts b/src/commands/agent/preview/send.ts index 36a9c67a..92dac6da 100644 --- a/src/commands/agent/preview/send.ts +++ b/src/commands/agent/preview/send.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages, SfError } from '@salesforce/core'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError, Lifecycle, EnvironmentVariable } from '@salesforce/core'; import { Agent } from '@salesforce/agents'; import { getCachedSessionIds, validatePreviewSession } from '../../../previewSessionStore.js'; @@ -32,6 +32,18 @@ export default class AgentPreviewSend extends SfCommand public static readonly examples = messages.getMessages('examples'); public static readonly requiresProject = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Message sent successfully and agent response received.', + 'NotFound (2)': 'Agent not found, or no preview session exists for this agent.', + 'PreviewSendFailed (4)': 'Failed to send message or receive response from the preview session.', + 'SessionAmbiguous (5)': 'Multiple preview sessions found; specify --session-id to choose one.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -57,31 +69,75 @@ export default class AgentPreviewSend extends SfCommand public async run(): Promise { const { flags } = await this.parse(AgentPreviewSend); - const conn = flags['target-org'].getConnection(flags['api-version']); + const agentIdentifier = flags['authoring-bundle'] ?? flags['api-name']!; - const agent = flags['authoring-bundle'] - ? await Agent.init({ connection: conn, project: this.project!, aabName: flags['authoring-bundle'] }) - : await Agent.init({ connection: conn, project: this.project!, apiNameOrId: flags['api-name']! }); + // Initialize agent with error tracking + let agent; + try { + agent = flags['authoring-bundle'] + ? await Agent.init({ connection: conn, project: this.project!, aabName: flags['authoring-bundle'] }) + : await Agent.init({ connection: conn, project: this.project!, apiNameOrId: flags['api-name']! }); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_agent_not_found' }); + throw new SfError(messages.getMessage('error.agentNotFound', [agentIdentifier]), 'AgentNotFound', [], 2, wrapped); + } + // Get or validate session ID let sessionId = flags['session-id']; if (sessionId === undefined) { const cached = await getCachedSessionIds(this.project!, agent); if (cached.length === 0) { - throw new SfError(messages.getMessage('error.noSession'), 'PreviewSessionNotFound'); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_no_session' }); + throw new SfError(messages.getMessage('error.noSession'), 'PreviewSessionNotFound', [], 2); } if (cached.length > 1) { + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_multiple_sessions' }); throw new SfError( messages.getMessage('error.multipleSessions', [cached.join(', ')]), - 'PreviewSessionAmbiguous' + 'PreviewSessionAmbiguous', + [], + 5 ); } sessionId = cached[0]; } + agent.setSessionId(sessionId); - await validatePreviewSession(agent); - const response = await agent.preview.send(flags.utterance); + // Validate session + try { + await validatePreviewSession(agent); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_session_invalid' }); + throw new SfError( + messages.getMessage('error.sessionInvalid', [sessionId]), + 'PreviewSessionInvalid', + [], + 2, + wrapped + ); + } + + // Send message with error tracking + let response; + try { + response = await agent.preview.send(flags.utterance); + } catch (error) { + const wrapped = SfError.wrap(error); + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_failed' }); + throw new SfError( + messages.getMessage('error.sendFailed', [wrapped.message]), + 'PreviewSendFailed', + [wrapped.message], + 4, + wrapped + ); + } + + await Lifecycle.getInstance().emitTelemetry({ eventName: 'agent_preview_send_success' }); this.log(response.messages[0].message); return { messages: response.messages ?? [] }; } diff --git a/src/commands/agent/preview/sessions.ts b/src/commands/agent/preview/sessions.ts index 95cb641f..ab15d221 100644 --- a/src/commands/agent/preview/sessions.ts +++ b/src/commands/agent/preview/sessions.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { SfCommand } from '@salesforce/sf-plugins-core'; +import { SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; import { Messages } from '@salesforce/core'; import { listCachedSessions } from '../../../previewSessionStore.js'; @@ -29,6 +29,10 @@ export default class AgentPreviewSessions extends SfCommand { const entries = await listCachedSessions(this.project!); const rows: AgentPreviewSessionsResult = []; diff --git a/src/commands/agent/preview/start.ts b/src/commands/agent/preview/start.ts index 2e7d0bf4..738f40a7 100644 --- a/src/commands/agent/preview/start.ts +++ b/src/commands/agent/preview/start.ts @@ -14,10 +14,11 @@ * limitations under the License. */ -import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; -import { Lifecycle, Messages } from '@salesforce/core'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Lifecycle, Messages, SfError, EnvironmentVariable } from '@salesforce/core'; import { Agent, ProductionAgent, ScriptAgent } from '@salesforce/agents'; import { createCache } from '../../../previewSessionStore.js'; +import { COMPILATION_API_EXIT_CODES } from '../../../common.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.preview.start'); @@ -32,6 +33,20 @@ export default class AgentPreviewStart extends SfCommand { public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Test created and deployed successfully.', + 'Failed (1)': 'Test validation errors or metadata format issues.', + 'NotFound (2)': 'Test spec file not found or org connection failed.', + 'DeploymentFailed (4)': 'Deployment failed due to API or network errors.', + }); + public static readonly flags = { ...makeFlags(FLAGGABLE_PROMPTS), 'target-org': Flags.requiredOrg(), @@ -163,10 +175,35 @@ export default class AgentTestCreate extends SfCommand { return Promise.resolve(); }); - const { path, contents } = await AgentTest.create(connection, apiName, spec, { - outputDir: join('force-app', 'main', 'default', 'aiEvaluationDefinitions'), - preview: flags.preview, - }); + let path; + let contents; + try { + const result = await AgentTest.create(connection, apiName, spec, { + outputDir: join('force-app', 'main', 'default', 'aiEvaluationDefinitions'), + preview: flags.preview, + }); + path = result.path; + contents = result.contents; + } catch (error) { + const wrapped = SfError.wrap(error); + + // Check for file not found errors + if ( + wrapped.message.toLowerCase().includes('not found') || + wrapped.message.toLowerCase().includes('enoent') || + wrapped.code === 'ENOENT' + ) { + throw new SfError(`Test spec file not found: ${spec}`, 'SpecFileNotFound', [], 2, wrapped); + } + + // Check for deployment failures (API/network) + if (wrapped.message.toLowerCase().includes('deploy') || wrapped.message.toLowerCase().includes('api')) { + throw new SfError(`Deployment failed: ${wrapped.message}`, 'DeploymentFailed', [wrapped.message], 4, wrapped); + } + + // Other errors (validation, format issues) use exit 1 + throw wrapped; + } if (flags.preview) { this.mso?.skipTo(AgentTestCreateLifecycleStages.Done); diff --git a/src/commands/agent/test/list.ts b/src/commands/agent/test/list.ts index 88fedfa1..b4382b7e 100644 --- a/src/commands/agent/test/list.ts +++ b/src/commands/agent/test/list.ts @@ -15,8 +15,8 @@ */ import { AgentTest, type AvailableDefinition } from '@salesforce/agents'; -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, EnvironmentVariable, SfError } from '@salesforce/core'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.test.list'); @@ -28,6 +28,16 @@ export default class AgentTestList extends SfCommand { public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Agent tests listed successfully.', + 'Failed (4)': 'Failed to retrieve agent tests due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -36,7 +46,20 @@ export default class AgentTestList extends SfCommand { public async run(): Promise { const { flags } = await this.parse(AgentTestList); - const results = await AgentTest.list(flags['target-org'].getConnection(flags['api-version'])); + let results; + try { + results = await AgentTest.list(flags['target-org'].getConnection(flags['api-version'])); + } catch (error) { + const wrapped = SfError.wrap(error); + throw new SfError( + `Failed to retrieve agent tests: ${wrapped.message}`, + 'ListRetrievalFailed', + [wrapped.message], + 4, + wrapped + ); + } + this.table({ data: results, columns: [ diff --git a/src/commands/agent/test/results.ts b/src/commands/agent/test/results.ts index 0dfbffd5..69cfe77e 100644 --- a/src/commands/agent/test/results.ts +++ b/src/commands/agent/test/results.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; import { AgentTester, AgentTestResultsResponse } from '@salesforce/agents'; import { resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js'; import { handleTestResults } from '../../../handleTestResults.js'; @@ -30,6 +30,17 @@ export default class AgentTestResults extends SfCommand public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Results retrieved successfully. Test results (passed/failed) are in the output.', + 'NotFound (2)': 'Job ID not found or invalid.', + 'Failed (4)': 'Failed to retrieve results due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -47,7 +58,32 @@ export default class AgentTestResults extends SfCommand const { flags } = await this.parse(AgentTestResults); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const response = await agentTester.results(flags['job-id']); + + let response; + try { + response = await agentTester.results(flags['job-id']); + } catch (error) { + const wrapped = SfError.wrap(error); + + // Check for job not found errors + if ( + wrapped.message.toLowerCase().includes('not found') || + wrapped.message.toLowerCase().includes('invalid') || + wrapped.code === 'ENOENT' + ) { + throw new SfError(`Job ID '${flags['job-id']}' not found or invalid.`, 'JobNotFound', [], 2, wrapped); + } + + // API/network failures + throw new SfError( + `Failed to retrieve results: ${wrapped.message}`, + 'ResultsRetrievalFailed', + [wrapped.message], + 4, + wrapped + ); + } + await handleTestResults({ id: flags['job-id'], format: flags['result-format'], diff --git a/src/commands/agent/test/resume.ts b/src/commands/agent/test/resume.ts index 65a40269..bca01e77 100644 --- a/src/commands/agent/test/resume.ts +++ b/src/commands/agent/test/resume.ts @@ -14,9 +14,10 @@ * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; import { AgentTester } from '@salesforce/agents'; +import { CLIError } from '@oclif/core/errors'; import { AgentTestCache } from '../../../agentTestCache.js'; import { TestStages } from '../../../testStages.js'; import { AgentTestRunResult, resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js'; @@ -30,6 +31,18 @@ export default class AgentTestResume extends SfCommand { public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Test completed successfully (with test results in the output).', + 'Failed (1)': 'Tests encountered execution errors (test cases with ERROR status).', + 'NotFound (2)': 'Job ID not found or invalid.', + 'OperationFailed (4)': 'Failed to poll test due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -55,26 +68,65 @@ export default class AgentTestResume extends SfCommand { verbose: verboseFlag, }; + private mso: TestStages | undefined; + public async run(): Promise { const { flags } = await this.parse(AgentTestResume); const agentTestCache = await AgentTestCache.create(); - const { name, runId, outputDir, resultFormat } = agentTestCache.useIdOrMostRecent( - flags['job-id'], - flags['use-most-recent'] - ); + let name; + let runId; + let outputDir; + let resultFormat; + + try { + const cacheEntry = agentTestCache.useIdOrMostRecent(flags['job-id'], flags['use-most-recent']); + name = cacheEntry.name; + runId = cacheEntry.runId; + outputDir = cacheEntry.outputDir; + resultFormat = cacheEntry.resultFormat; + } catch (e) { + const wrapped = SfError.wrap(e); + + // Check for job not found + if ( + wrapped.message.toLowerCase().includes('not found') || + wrapped.message.toLowerCase().includes('no test') || + wrapped.message.toLowerCase().includes('invalid') + ) { + throw new SfError(`Job ID '${flags['job-id'] ?? 'most recent'}' not found.`, 'JobNotFound', [], 2, wrapped); + } - const mso = new TestStages({ + throw wrapped; + } + + this.mso = new TestStages({ title: `Agent Test Run: ${name ?? runId}`, jsonEnabled: this.jsonEnabled(), }); - mso.start({ id: runId }); + this.mso.start({ id: runId }); const agentTester = new AgentTester(flags['target-org'].getConnection(flags['api-version'])); - const { completed, response } = await mso.poll(agentTester, runId, flags.wait); + let completed; + let response; + try { + const pollResult = await this.mso.poll(agentTester, runId, flags.wait); + completed = pollResult.completed; + response = pollResult.response; + } catch (error) { + const wrapped = SfError.wrap(error); + throw new SfError( + `Failed to poll test results: ${wrapped.message}`, + 'TestPollFailed', + [wrapped.message], + 4, + wrapped + ); + } + if (completed) await agentTestCache.removeCacheEntry(runId); - mso.stop(); + this.mso.stop(); await handleTestResults({ id: runId, @@ -85,6 +137,18 @@ export default class AgentTestResume extends SfCommand { verbose: flags.verbose, }); + // Set exit code to 1 only for execution errors (tests couldn't run properly) + // Test assertion failures are business logic and should not affect exit code + if (response?.testCases.some((tc) => tc.status === 'ERROR')) { + process.exitCode = 1; + } + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion return { ...response!, runId, status: 'COMPLETED' }; } + + protected catch(error: Error | SfError | CLIError): Promise { + this.mso?.error(); + return super.catch(error); + } } diff --git a/src/commands/agent/test/run-eval.ts b/src/commands/agent/test/run-eval.ts index f364c434..d2569526 100644 --- a/src/commands/agent/test/run-eval.ts +++ b/src/commands/agent/test/run-eval.ts @@ -15,8 +15,8 @@ */ import { readFile } from 'node:fs/promises'; -import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; -import { Messages, Org } from '@salesforce/core'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages, Org, SfError } from '@salesforce/core'; import { type EvalPayload, normalizePayload, splitIntoBatches } from '../../../evalNormalizer.js'; import { type EvalApiResponse, formatResults, type ResultFormat } from '../../../evalFormatter.js'; import { resultFormatFlag } from '../../../flags.js'; @@ -157,6 +157,18 @@ export default class AgentTestRunEval extends SfCommand { public static state = 'beta'; public static readonly hidden = true; + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Tests completed successfully. Test results (passed/failed) are in the JSON output.', + 'Failed (1)': "Tests encountered execution errors (tests couldn't run properly).", + 'NotFound (2)': 'Agent not found, spec file not found, or invalid agent name.', + 'OperationFailed (4)': 'Failed to execute tests due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -203,7 +215,12 @@ export default class AgentTestRunEval extends SfCommand { // If we got here, it's valid content } catch { // Not valid content, must be a file path - read it - rawContent = await readFile(flags.spec, 'utf-8'); + try { + rawContent = await readFile(flags.spec, 'utf-8'); + } catch (e) { + const wrapped = SfError.wrap(e); + throw new SfError(`Spec file not found: ${flags.spec}`, 'SpecFileNotFound', [], 2, wrapped); + } } // 2. Detect format and parse @@ -235,7 +252,17 @@ export default class AgentTestRunEval extends SfCommand { // 3. If --api-name (or auto-inferred from YAML), resolve IDs and inject if (agentApiName) { - const { agentId, versionId } = await resolveAgent(org, agentApiName); + let agentId; + let versionId; + try { + const resolved = await resolveAgent(org, agentApiName); + agentId = resolved.agentId; + versionId = resolved.versionId; + } catch (e) { + const wrapped = SfError.wrap(e); + throw new SfError(`Agent '${agentApiName}' not found.`, 'AgentNotFound', [], 2, wrapped); + } + for (const test of payload.tests) { for (const step of test.steps) { if (step.type === 'agent.create_session') { @@ -260,7 +287,19 @@ export default class AgentTestRunEval extends SfCommand { const batches = splitIntoBatches(payload.tests, batchSize); // 7. Execute batches - const allResults = await executeBatches(org, batches, (msg) => this.log(msg)); + let allResults; + try { + allResults = await executeBatches(org, batches, (msg) => this.log(msg)); + } catch (e) { + const wrapped = SfError.wrap(e); + throw new SfError( + `Failed to execute tests: ${wrapped.message}`, + 'TestExecutionFailed', + [wrapped.message], + 4, + wrapped + ); + } const mergedResponse: EvalApiResponse = { results: allResults as EvalApiResponse['results'] }; @@ -272,8 +311,9 @@ export default class AgentTestRunEval extends SfCommand { // 10. Build structured result for --json const { summary, testSummaries } = buildResultSummary(mergedResponse); - // Set exit code to 1 if any tests failed - if (summary.failed > 0 || summary.errors > 0) { + // Set exit code to 1 only for execution errors (tests couldn't run) + // Test failures (assertions failed) are business logic and should not affect exit code + if (summary.errors > 0) { process.exitCode = 1; } diff --git a/src/commands/agent/test/run.ts b/src/commands/agent/test/run.ts index 9376965e..330223c6 100644 --- a/src/commands/agent/test/run.ts +++ b/src/commands/agent/test/run.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages, SfError } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages, SfError } from '@salesforce/core'; import { AgentTester, AgentTestStartResponse } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; import { CLIError } from '@oclif/core/errors'; @@ -62,6 +62,18 @@ export default class AgentTestRun extends SfCommand { public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Test started successfully (without --wait), or test completed successfully (with --wait).', + 'Failed (1)': 'Tests encountered execution errors (test cases with ERROR status when using --wait).', + 'NotFound (2)': 'Test definition not found or invalid test name.', + 'OperationFailed (4)': 'Failed to start or poll test due to API or network errors.', + }); + public static readonly flags = { 'target-org': Flags.requiredOrg(), 'api-version': Flags.orgApiVersion(), @@ -101,12 +113,23 @@ export default class AgentTestRun extends SfCommand { response = await agentTester.start(apiName); } catch (e) { const wrapped = SfError.wrap(e); - if (wrapped.message.includes('Invalid AiEvalDefinitionVersion identifier')) { - wrapped.actions = [ - `Try running "sf agent test list -o ${flags['target-org'].getUsername() ?? ''}" to see available options`, - ]; + + // Check for test definition not found + if ( + wrapped.message.includes('Invalid AiEvalDefinitionVersion identifier') || + wrapped.message.toLowerCase().includes('not found') + ) { + throw new SfError( + `Test definition '${apiName}' not found.`, + 'TestNotFound', + [`Try running "sf agent test list -o ${flags['target-org'].getUsername() ?? ''}" to see available options`], + 2, + wrapped + ); } - throw wrapped; + + // API/network failures + throw new SfError(`Failed to start test: ${wrapped.message}`, 'TestStartFailed', [wrapped.message], 4, wrapped); } this.mso.update({ id: response.runId }); @@ -115,7 +138,23 @@ export default class AgentTestRun extends SfCommand { await agentTestCache.createCacheEntry(response.runId, apiName, flags['output-dir'], flags['result-format']); if (flags.wait?.minutes) { - const { completed, response: detailsResponse } = await this.mso.poll(agentTester, response.runId, flags.wait); + let completed; + let detailsResponse; + try { + const pollResult = await this.mso.poll(agentTester, response.runId, flags.wait); + completed = pollResult.completed; + detailsResponse = pollResult.response; + } catch (error) { + const wrapped = SfError.wrap(error); + throw new SfError( + `Failed to poll test results: ${wrapped.message}`, + 'TestPollFailed', + [wrapped.message], + 4, + wrapped + ); + } + if (completed) await agentTestCache.removeCacheEntry(response.runId); this.mso.stop(); @@ -129,6 +168,13 @@ export default class AgentTestRun extends SfCommand { verbose: flags.verbose, }); + // Set exit code to 1 only for execution errors (tests couldn't run properly) + // Test assertion failures are business logic and should not affect exit code + if (detailsResponse?.testCases.some((tc) => tc.status === 'ERROR')) { + process.exitCode = 1; + } + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion return { ...detailsResponse!, status: 'COMPLETED', runId: response.runId }; } else { this.mso.stop(); diff --git a/src/commands/agent/validate/authoring-bundle.ts b/src/commands/agent/validate/authoring-bundle.ts index b179f48b..c5347db8 100644 --- a/src/commands/agent/validate/authoring-bundle.ts +++ b/src/commands/agent/validate/authoring-bundle.ts @@ -13,8 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { EnvironmentVariable, Messages } from '@salesforce/core'; import { MultiStageOutput } from '@oclif/multi-stage-output'; import { Agent } from '@salesforce/agents'; import { colorize } from '@oclif/core/ux'; @@ -35,6 +35,19 @@ export default class AgentValidateAuthoringBundle extends SfCommand { - /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ const historyDir = await agent.getHistoryDir(); const metaPath = join(historyDir, SESSION_META_FILE); - /* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ const meta: SessionMeta = { displayName: options?.displayName }; await writeFile(metaPath, JSON.stringify(meta), 'utf-8'); } @@ -47,10 +45,8 @@ export async function createCache( * Throws SfError if the session marker is not found. */ export async function validatePreviewSession(agent: ScriptAgent | ProductionAgent): Promise { - /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ const historyDir = await agent.getHistoryDir(); const metaPath = join(historyDir, SESSION_META_FILE); - /* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ try { await readFile(metaPath, 'utf-8'); } catch { @@ -66,10 +62,8 @@ export async function validatePreviewSession(agent: ScriptAgent | ProductionAgen * Call after ending the session. Caller must set sessionId on the agent before calling. */ export async function removeCache(agent: ScriptAgent | ProductionAgent): Promise { - /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ const historyDir = await agent.getHistoryDir(); const metaPath = join(historyDir, SESSION_META_FILE); - /* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-argument */ try { await unlink(metaPath); } catch { diff --git a/test/nuts/agent.test.run-eval.nut.ts b/test/nuts/agent.test.run-eval.nut.ts index e21d4b30..12aa7353 100644 --- a/test/nuts/agent.test.run-eval.nut.ts +++ b/test/nuts/agent.test.run-eval.nut.ts @@ -39,105 +39,98 @@ describe('agent test run-eval', function () { describe('run-eval with JSON file', () => { it('should run evaluation with JSON payload file', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; - - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); - expect(output?.result.tests.length).to.be.greaterThan(0); - expect(output?.result.summary).to.be.ok; - expect(output?.result.summary.passed).to.be.a('number'); - expect(output?.result.summary.failed).to.be.a('number'); - expect(output?.result.summary.scored).to.be.a('number'); - expect(output?.result.summary.errors).to.be.a('number'); + const result = execCmd(command, { ensureExitCode: 0 }); + + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); + expect(result.jsonOutput?.result.summary).to.be.ok; + expect(result.jsonOutput?.result.summary.passed).to.be.a('number'); + expect(result.jsonOutput?.result.summary.failed).to.be.a('number'); + expect(result.jsonOutput?.result.summary.scored).to.be.a('number'); + expect(result.jsonOutput?.result.summary.errors).to.be.a('number'); }); it('should run evaluation with normalized payload', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result.tests[0]).to.be.ok; - expect(output?.result.tests[0].id).to.equal('test-topic-routing'); - expect(output?.result.tests[0].status).to.be.oneOf(['passed', 'failed']); - expect(output?.result.tests[0].evaluations).to.be.an('array'); + expect(result.jsonOutput?.result.tests[0]).to.be.ok; + expect(result.jsonOutput?.result.tests[0].id).to.equal('test-topic-routing'); + expect(result.jsonOutput?.result.tests[0].status).to.be.oneOf(['passed', 'failed']); + expect(result.jsonOutput?.result.tests[0].evaluations).to.be.an('array'); }); }); describe('run-eval with YAML file', () => { it('should run evaluation with YAML test spec file', async () => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); - expect(output?.result.tests.length).to.be.greaterThan(0); - expect(output?.result.summary).to.be.ok; + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); + expect(result.jsonOutput?.result.summary).to.be.ok; }); it('should auto-infer agent name from YAML subjectName', async () => { const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); // Should succeed without explicit --api-name flag - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); }); it('should handle YAML spec with contextVariables', async () => { const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); // Verify the command succeeds with contextVariables - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); - expect(output?.result.tests.length).to.be.greaterThan(0); - expect(output?.result.summary).to.be.ok; + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0); + expect(result.jsonOutput?.result.summary).to.be.ok; }); }); describe('run-eval with flags', () => { it('should respect --no-normalize flag', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + const result = execCmd(command, { ensureExitCode: 1 }); - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); }); it('should use custom batch size', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result).to.be.ok; - expect(output?.result.tests).to.be.an('array'); + expect(result.jsonOutput?.result).to.be.ok; + expect(result.jsonOutput?.result.tests).to.be.an('array'); }); it('should support different result formats', async () => { - // Test human format (default) - don't enforce exit code since tests may fail + // Test human format (default) const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`; - const humanOutput = execCmd(humanCommand).shellOutput.stdout; + const humanResult = execCmd(humanCommand, { ensureExitCode: 0 }); - expect(humanOutput).to.be.ok; - expect(humanOutput).to.be.a('string'); + expect(humanResult.shellOutput.stdout).to.be.ok; + expect(humanResult.shellOutput.stdout).to.be.a('string'); - // Test tap format - don't enforce exit code since tests may fail + // Test tap format const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`; - const tapOutput = execCmd(tapCommand).shellOutput.stdout; + const tapResult = execCmd(tapCommand, { ensureExitCode: 0 }); - expect(tapOutput).to.include('TAP version'); + expect(tapResult.shellOutput.stdout).to.include('TAP version'); - // Test junit format - don't enforce exit code since tests may fail + // Test junit format const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`; - const junitOutput = execCmd(junitCommand).shellOutput.stdout; + const junitResult = execCmd(junitCommand, { ensureExitCode: 0 }); - expect(junitOutput).to.include(' { it('should include test summaries with correct structure', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - expect(output?.result.tests).to.be.an('array'); - const firstTest = output?.result.tests[0]; + expect(result.jsonOutput?.result.tests).to.be.an('array'); + const firstTest = result.jsonOutput?.result.tests[0]; expect(firstTest).to.have.property('id'); expect(firstTest).to.have.property('status'); expect(firstTest).to.have.property('evaluations'); @@ -198,10 +190,9 @@ describe('agent test run-eval', function () { it('should include summary with all metrics', async () => { const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`; - // Don't enforce exit code 0 since the command exits with 1 if tests fail - const output = execCmd(command).jsonOutput; + const result = execCmd(command, { ensureExitCode: 0 }); - const summary = output?.result.summary; + const summary = result.jsonOutput?.result.summary; expect(summary).to.have.property('passed'); expect(summary).to.have.property('failed'); expect(summary).to.have.property('scored');