From 82aad2489fa4121e1a1348c5d389c140d4b42576 Mon Sep 17 00:00:00 2001 From: sarthak Date: Tue, 5 May 2026 14:02:49 +0530 Subject: [PATCH] Refactor API documentation for test executions - Removed the "Call Executions" card from the API index page. - Updated request and response body parameters in the "Delete Test Executions" documentation to use snake_case. - Removed unnecessary request body section in the "Cancel Execution" documentation. - Added new documentation for "Get Call Execution Details" API endpoint. - Deleted outdated "Get Eval Explanation Summary" documentation. - Updated KPI metrics in "Get KPIs" documentation to use snake_case. - Enhanced "Get Test Execution Details" documentation to include pagination and column configuration. - Updated request and response parameters in "Rerun Calls" documentation to use snake_case and clarified descriptions. --- src/components/docs/ApiExplorer.astro | 59 +++++-- src/components/docs/ApiPlayground.astro | 8 +- src/lib/api-navigation.ts | 19 +-- src/lib/navigation.ts | 9 +- .../call-executions/getsessioncomparison.mdx | 84 ---------- src/pages/docs/api/index.mdx | 3 - .../api/run-tests/deletetestexecutions.mdx | 18 +-- .../api/test-executions/cancelexecution.mdx | 6 - .../getcallexecutiondetails.mdx | 44 +++++- .../getevalexplanationsummary.mdx | 58 ------- .../docs/api/test-executions/getkpis.mdx | 24 +-- .../gettestexecutiondetails.mdx | 147 +++++++++++------- .../docs/api/test-executions/reruncalls.mdx | 53 ++++--- 13 files changed, 232 insertions(+), 300 deletions(-) delete mode 100644 src/pages/docs/api/call-executions/getsessioncomparison.mdx rename src/pages/docs/api/{call-executions => test-executions}/getcallexecutiondetails.mdx (75%) delete mode 100644 src/pages/docs/api/test-executions/getevalexplanationsummary.mdx diff --git a/src/components/docs/ApiExplorer.astro b/src/components/docs/ApiExplorer.astro index ed80aaa3..9f0e3398 100644 --- a/src/components/docs/ApiExplorer.astro +++ b/src/components/docs/ApiExplorer.astro @@ -9,16 +9,27 @@ * target pages and extracts .apg data to update the drawer in-place (no page reload). */ import { tabNavigation } from '../../lib/navigation.ts'; +import { apiNavigation } from '../../lib/api-navigation.ts'; +// Build a href → method lookup from api-navigation.ts (source of truth for HTTP methods) +const methodByHref: Record = {}; +for (const group of apiNavigation) { + for (const item of group.items) { + if (item.href && item.method) { + methodByHref[item.href] = item.method; + } + } +} // Extract the API tab navigation at build time const apiTab = tabNavigation.find(t => t.tab === 'API'); const apiGroups = apiTab ? apiTab.groups : []; // Flatten the API groups into a structure suitable for the sidebar: -// { group: string, items: { title: string, href: string, method?: string }[] }[] +// { group: string, items: { title: string, href: string, method: string }[] }[] interface SidebarEndpoint { title: string; href: string; + method: string; } interface SidebarGroup { group: string; @@ -29,7 +40,7 @@ function flattenItems(items: any[]): SidebarEndpoint[] { const result: SidebarEndpoint[] = []; for (const item of items) { if (item.href && item.href !== '/docs/api') { - result.push({ title: item.title, href: item.href }); + result.push({ title: item.title, href: item.href, method: methodByHref[item.href] || 'GET' }); } if (item.items) { result.push(...flattenItems(item.items)); @@ -476,8 +487,8 @@ const sidebarDataJson = JSON.stringify(sidebarGroups); for (var j = 0; j < groupItems.length; j++) { var ep = groupItems[j]; - // Infer method from title - var method = inferMethodFromTitle(ep.title); + // Use method from api-navigation.ts (baked in at build time), fall back to title inference + var method = ep.method || inferMethodFromTitle(ep.title); var isActive = ep.href === window.location.pathname.replace(/\/$/, ''); html += ''; html += '' + shortMethod(method) + ''; @@ -494,7 +505,7 @@ const sidebarDataJson = JSON.stringify(sidebarGroups); function inferMethodFromTitle(title) { var t = title.toLowerCase(); - if (t.indexOf('create') === 0 || t.indexOf('add') === 0 || t.indexOf('submit') === 0 || t.indexOf('apply') === 0 || t.indexOf('bulk') === 0 || t.indexOf('upload') === 0 || t.indexOf('run') === 0 || t.indexOf('execute') === 0 || t.indexOf('send') === 0 || t.indexOf('generate') === 0 || t.indexOf('clone') === 0) return 'POST'; + if (t.indexOf('create') === 0 || t.indexOf('add') === 0 || t.indexOf('submit') === 0 || t.indexOf('apply') === 0 || t.indexOf('bulk') === 0 || t.indexOf('upload') === 0 || t.indexOf('run') === 0 || t.indexOf('execute') === 0 || t.indexOf('send') === 0 || t.indexOf('generate') === 0 || t.indexOf('clone') === 0 || t.indexOf('rerun') === 0 || t.indexOf('cancel') === 0 || t.indexOf('duplicate') === 0 || t.indexOf('restore') === 0 || t.indexOf('pause') === 0 || t.indexOf('unpause') === 0 || t.indexOf('release') === 0 || t.indexOf('assign') === 0 || t.indexOf('complete') === 0 || t.indexOf('skip') === 0 || t.indexOf('fetch') === 0 || t.indexOf('export') === 0) return 'POST'; if (t.indexOf('update') === 0 || t.indexOf('edit') === 0 || t.indexOf('merge') === 0) return 'PATCH'; if (t.indexOf('delete') === 0 || t.indexOf('remove') === 0) return 'DELETE'; return 'GET'; @@ -680,13 +691,29 @@ const sidebarDataJson = JSON.stringify(sidebarGroups); function buildBodyForm(body, method) { if (!bodySection || !bodyFields) return; - if (!body || typeof body !== 'object' || Array.isArray(body) || !/POST|PUT|PATCH/.test(method)) { + if (!/POST|PUT|PATCH|DELETE/.test(method)) { bodySection.style.display = 'none'; bodyFields.innerHTML = ''; if (bodyIntro) bodyIntro.style.display = 'none'; return; } + // Merge ParamField body metadata keys into body so fields documented + // via but missing from requestBody still appear. + body = body && typeof body === 'object' && !Array.isArray(body) ? Object.assign({}, body) : {}; + for (var mk in currentBodyMeta) { + if (currentBodyMeta.hasOwnProperty(mk) && mk.indexOf('.') === -1 && !(mk in body)) { + var mt = currentBodyMeta[mk].type || 'string'; + if (/array|list/i.test(mt)) body[mk] = []; + else if (/object/i.test(mt)) body[mk] = {}; + else if (/bool/i.test(mt)) body[mk] = false; + else if (/int|number|double|float/i.test(mt)) body[mk] = 0; + else body[mk] = ''; + } + } + // Keep currentReqBody in sync so optional picker and code gen use merged body + currentReqBody = body; + var keys = Object.keys(body); if (keys.length === 0) { bodySection.style.display = 'none'; @@ -2011,7 +2038,7 @@ const sidebarDataJson = JSON.stringify(sidebarGroups); } else { lines.push(' -H "Authorization: Bearer ' + authInfo.token + '"'); } - if (body && /POST|PUT|PATCH/.test(method)) { + if (body && /POST|PUT|PATCH|DELETE/.test(method)) { lines[lines.length - 1] += ' \\'; lines.push(' -H "Content-Type: application/json" \\'); lines.push(" -d '" + JSON.stringify(body, null, 2).replace(/'/g, "'\\''") + "'"); @@ -2026,7 +2053,7 @@ const sidebarDataJson = JSON.stringify(sidebarGroups); if (parts.length > 1 && method === 'GET') sdkMethod = 'get'; var apiKeyVal = isApiKey ? authInfo.apiKey : authInfo.token; var lines = ['from fi.client import FutureAGI', '', 'client = FutureAGI(api_key="' + apiKeyVal + '")', '']; - if (body && typeof body === 'object' && /POST|PUT|PATCH/.test(method)) { + if (body && typeof body === 'object' && /POST|PUT|PATCH|DELETE/.test(method)) { lines.push('response = client.' + resource + '.' + sdkMethod + '('); var bKeys = Object.keys(body); for (var ki = 0; ki < bKeys.length; ki++) { @@ -2047,16 +2074,16 @@ const sidebarDataJson = JSON.stringify(sidebarGroups); headerStr = ' headers={"Authorization": "Bearer ' + authInfo.token + '"},'; } var lines = ['import requests', '', 'response = requests.' + method.toLowerCase() + '(', ' "' + url + '",', headerStr]; - if (body && /POST|PUT|PATCH/.test(method)) lines.push(' json=' + JSON.stringify(body, null, 4) + ','); + if (body && /POST|PUT|PATCH|DELETE/.test(method)) lines.push(' json=' + JSON.stringify(body, null, 4) + ','); lines.push(')'); lines.push('print(response.json())'); return lines.join('\n'); } if (lang === 'go') { var lines = ['package main', '', 'import (', ' "bytes"', ' "fmt"', ' "io"', ' "net/http"']; - if (body && /POST|PUT|PATCH/.test(method)) lines.push(' "encoding/json"'); + if (body && /POST|PUT|PATCH|DELETE/.test(method)) lines.push(' "encoding/json"'); lines.push(')', ''); lines.push('func main() {'); - if (body && /POST|PUT|PATCH/.test(method)) { + if (body && /POST|PUT|PATCH|DELETE/.test(method)) { lines.push(' body, _ := json.Marshal(' + JSON.stringify(body) + ')'); lines.push(' req, _ := http.NewRequest("' + method + '", "' + url + '", bytes.NewBuffer(body))'); } else { @@ -2089,7 +2116,7 @@ const sidebarDataJson = JSON.stringify(sidebarGroups); builderLines.push(' .header("Authorization", "Bearer ' + authInfo.token + '")'); } builderLines.push(' .header("Content-Type", "application/json")'); - if (body && /POST|PUT|PATCH/.test(method)) { + if (body && /POST|PUT|PATCH|DELETE/.test(method)) { builderLines.push(' .' + method + '(HttpRequest.BodyPublishers.ofString(' + JSON.stringify(JSON.stringify(body)) + '))'); } else if (method === 'DELETE') { builderLines.push(' .DELETE()'); @@ -2105,7 +2132,7 @@ const sidebarDataJson = JSON.stringify(sidebarGroups); } if (lang === 'php') { var lines = [' - - - - Your Future AGI API key used to authenticate requests. You can find and manage your API keys in the [Dashboard](https://app.futureagi.com) under Settings. - - - Your Future AGI secret key, used alongside the API key for request authentication. This is generated when you create an API key in the [Dashboard](https://app.futureagi.com). - - - - - - The call execution ID. Must have at least one rerun snapshot. - - - - - Performance metrics for current and previous sessions. - - Current execution metrics. - Previous rerun snapshot metrics. - - Transcripts for current and previous sessions. - - Current session transcript with `role` and `content`. - Previous rerun snapshot transcript. - - Audio recording URLs. Voice only. - - Current execution recording URL. - Previous rerun snapshot recording URL. - - Evaluation results for both sessions. - - Current execution eval results. - Previous rerun snapshot eval results. - - - - - No rerun snapshot available for comparison. - Invalid or missing credentials. - Call execution not found. - Unexpected server error. - diff --git a/src/pages/docs/api/index.mdx b/src/pages/docs/api/index.mdx index 4eb23010..6bbc4508 100644 --- a/src/pages/docs/api/index.mdx +++ b/src/pages/docs/api/index.mdx @@ -50,9 +50,6 @@ Get your API key from the [Future AGI Dashboard](https://app.futureagi.com/setti Test execution tracking and analytics - - Individual call execution details - Dataset creation, modification, and data management diff --git a/src/pages/docs/api/run-tests/deletetestexecutions.mdx b/src/pages/docs/api/run-tests/deletetestexecutions.mdx index 6855a0c9..bd748e69 100644 --- a/src/pages/docs/api/run-tests/deletetestexecutions.mdx +++ b/src/pages/docs/api/run-tests/deletetestexecutions.mdx @@ -10,8 +10,8 @@ description: "Bulk-deletes test executions from a test run." parameters={[ {"name": "run_test_id", "in": "path", "required": true, "description": "UUID of the test run from which to delete test executions.", "type": "string"} ]} - requestBody={{"testExecutionIds": ["execution-uuid-1", "execution-uuid-2"], "selectAll": false}} - responseExample={{"message": "Successfully deleted 2 test execution(s).", "runTestId": "run-test-uuid", "deletedCount": 2, "deletedIds": ["execution-uuid-1", "execution-uuid-2"]}} + requestBody={{"test_execution_ids": ["execution-uuid-1", "execution-uuid-2"], "select_all": false}} + responseExample={{"message": "Successfully deleted 2 test execution(s).", "run_test_id": "run-test-uuid", "deleted_count": 2, "deleted_ids": ["execution-uuid-1", "execution-uuid-2"]}} responseStatus={200} responseStatusText="OK" /> @@ -32,19 +32,19 @@ description: "Bulk-deletes test executions from a test run." - - Array of test execution UUIDs to delete. Required when `selectAll` is `false`. Executions in `RUNNING`, `PENDING`, or `CANCELLING` status cannot be deleted. + + Array of test execution UUIDs to delete. Required when `select_all` is `false`. Executions in `RUNNING`, `PENDING`, or `CANCELLING` status cannot be deleted. - - When `true`, deletes all eligible executions, ignoring `testExecutionIds`. Defaults to `false`. + + When `true`, deletes all eligible executions, ignoring `test_execution_ids`. Defaults to `false`. Confirmation message with deletion count. - UUID of the parent test run. - Number of executions deleted. - UUIDs of the deleted executions. + UUID of the parent test run. + Number of executions deleted. + UUIDs of the deleted executions. diff --git a/src/pages/docs/api/test-executions/cancelexecution.mdx b/src/pages/docs/api/test-executions/cancelexecution.mdx index be2f3888..724759b0 100644 --- a/src/pages/docs/api/test-executions/cancelexecution.mdx +++ b/src/pages/docs/api/test-executions/cancelexecution.mdx @@ -10,7 +10,6 @@ description: "Cancels a test execution." parameters={[ {"name": "test_execution_id", "in": "path", "required": true, "description": "UUID of the test execution to cancel.", "type": "string"} ]} - requestBody={{}} responseExample={{ success: true, message: "Test execution cancellation initiated", @@ -35,11 +34,6 @@ description: "Cancels a test execution." - - - No body required. Send `{}`. - - Whether the cancellation was accepted. diff --git a/src/pages/docs/api/call-executions/getcallexecutiondetails.mdx b/src/pages/docs/api/test-executions/getcallexecutiondetails.mdx similarity index 75% rename from src/pages/docs/api/call-executions/getcallexecutiondetails.mdx rename to src/pages/docs/api/test-executions/getcallexecutiondetails.mdx index 33e8eea0..3f1632ca 100644 --- a/src/pages/docs/api/call-executions/getcallexecutiondetails.mdx +++ b/src/pages/docs/api/test-executions/getcallexecutiondetails.mdx @@ -35,26 +35,46 @@ description: "Retrieves a specific call execution." customer_name: "Jane Doe", call_summary: "Customer inquired about billing charges.", ended_reason: "customer_hangup", - simulatorAgentName: "Billing Simulator", - simulatorAgentId: "sim-agent-uuid", + simulator_agent_name: "Billing Simulator", + simulator_agent_id: "sim-agent-uuid", agent_definition_used_name: "Support Agent v2", agent_definition_used_id: "agent-def-uuid", tool_outputs: null, rerun_snapshots: [], + provider: "vapi", + phone_number: "+14155550100", + simulation_call_type: "voice", + processing_skipped: false, + processing_skip_reason: null, + is_snapshot: false, + snapshot_timestamp: null, + rerun_type: null, + original_call_execution_id: null, avg_agent_latency: 0.85, user_interruption_count: 1, user_interruption_rate: 0.05, user_wpm: 130, bot_wpm: 145, talk_ratio: 0.55, + agent_talk_percentage: 55.0, ai_interruption_count: 0, ai_interruption_rate: 0.0, avg_stop_time_after_interruption: 0.3, + total_tokens: null, + input_tokens: null, + output_tokens: null, + avg_latency_ms: null, + turn_count: null, + csat_score: null, stt_cost: 0.012, llm_cost: 0.045, tts_cost: 0.008, storage_cost: 0.002, - total_cost: 0.067 + total_cost: 0.067, + customer_cost_cents: null, + customer_cost_breakdown: null, + customer_latency_metrics: null, + customer_call_id: null }} responseStatus={200} responseStatusText="OK" @@ -101,12 +121,21 @@ description: "Retrieves a specific call execution." Simulated customer persona name. AI-generated conversation summary. Reason the call ended, e.g. `customer_hangup`, `agent_hangup`, `timeout`, `error`. - Simulator agent name. - UUID of the simulator agent. + Simulator agent name. + UUID of the simulator agent. Agent definition name. UUID of the agent definition. Tool call outputs from the conversation. Snapshots from previous reruns. + Telephony or chat provider used for this call, e.g. `vapi`, `retell`. + Phone number dialed for this call. Voice only. + Simulation mode: `voice` or `text`. + Whether post-call processing was skipped. + Reason processing was skipped, if applicable. + Whether this record is a rerun snapshot rather than the live call. + Timestamp when the snapshot was taken. + Type of the most recent rerun: `eval_only` or `call_and_eval`. `null` if never rerun. + UUID of the original call execution this is a snapshot of. Average agent response latency in seconds. Voice only. User interruption count. Voice only. Proportion of agent turns interrupted by user (0-1). Voice only. @@ -121,7 +150,12 @@ description: "Retrieves a specific call execution." Output tokens generated. Text only. Average response latency in milliseconds. Text only. Total conversation turns. Text only. + Percentage of conversation time the agent was talking (0-100). Voice only. Customer satisfaction score. Text only. + Cost of the call in cents as reported by the customer's telephony provider. + Detailed cost breakdown from the customer's provider. + Latency metrics as reported by the customer's provider. + Call ID assigned by the customer's telephony provider. Speech-to-text cost in USD. Voice only. LLM inference cost in USD. Text-to-speech cost in USD. Voice only. diff --git a/src/pages/docs/api/test-executions/getevalexplanationsummary.mdx b/src/pages/docs/api/test-executions/getevalexplanationsummary.mdx deleted file mode 100644 index 21bc87ce..00000000 --- a/src/pages/docs/api/test-executions/getevalexplanationsummary.mdx +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: "Get eval explanation summary" -description: "Retrieves the eval explanation summary for a test execution." ---- - - - - - - Your Future AGI API key used to authenticate requests. You can find and manage your API keys in the [Dashboard](https://app.futureagi.com) under Settings. - - - Your Future AGI secret key, used alongside the API key for request authentication. This is generated when you create an API key in the [Dashboard](https://app.futureagi.com). - - - - - - The test execution ID. Triggers async generation if not yet available. - - - - - Summary data and generation metadata. - - Summary with performance overview, issues, and recommendations. - ISO 8601 timestamp of last generation. - Generation status: `pending`, `running`, `completed`, or `failed`. - - Whether the request succeeded. - - - - Invalid or missing credentials. - Test execution not found. - Unexpected server error. - diff --git a/src/pages/docs/api/test-executions/getkpis.mdx b/src/pages/docs/api/test-executions/getkpis.mdx index 4e5cfd91..90b06778 100644 --- a/src/pages/docs/api/test-executions/getkpis.mdx +++ b/src/pages/docs/api/test-executions/getkpis.mdx @@ -14,14 +14,14 @@ description: "Retrieves KPI metrics for a test execution." total_calls: 50, avg_score: 8.2, avg_response: 1.15, - callsAttempted: 50, - connectedCalls: 47, - callsConnectedPercentage: 94.0, + calls_attempted: 50, + connected_calls: 47, + calls_connected_percentage: 94.0, failed_calls: 3, total_duration: 6250, agent_type: "voice", is_inbound: false, - scenarioGraphs: {}, + scenario_graphs: {}, avg_agent_latency: 0.92, avg_user_interruption_count: 1.4, avg_user_interruption_rate: 0.08, @@ -31,8 +31,8 @@ description: "Retrieves KPI metrics for a test execution." avg_ai_interruption_count: 0.3, avg_ai_interruption_rate: 0.02, avg_stop_time_after_interruption: 0.35, - agentTalkPercentage: 55.0, - customerTalkPercentage: 45.0, + agent_talk_percentage: 55.0, + customer_talk_percentage: 45.0, avg_tone_check: 8.7, avg_accuracy: 7.9 }} @@ -59,14 +59,14 @@ description: "Retrieves KPI metrics for a test execution." Total call executions. Average evaluation score across completed calls. Average response time in seconds. - Total calls initiated. - Calls that connected. - Percentage of calls that connected. + Total calls initiated. + Calls that connected. + Percentage of calls that connected. Calls that failed. Combined duration in seconds. `voice` or `text`. `true` for inbound, `false` for outbound. `null` for text agents. - Per-scenario performance data. + Per-scenario performance data. Average agent latency in seconds. Voice only. Average user interruptions per call. Voice only. Average user interruption rate (0-1). Voice only. @@ -76,8 +76,8 @@ description: "Retrieves KPI metrics for a test execution." Average agent interruptions per call. Voice only. Average agent interruption rate (0-1). Voice only. Average seconds to stop after interruption. Voice only. - Agent talk time percentage (0-100). Voice only. - Customer talk time percentage (0-100). Voice only. + Agent talk time percentage (0-100). Voice only. + Customer talk time percentage (0-100). Voice only. Average total tokens per call. Text only. Average input tokens per call. Text only. Average output tokens per call. Text only. diff --git a/src/pages/docs/api/test-executions/gettestexecutiondetails.mdx b/src/pages/docs/api/test-executions/gettestexecutiondetails.mdx index 0df874e3..f914bbbf 100644 --- a/src/pages/docs/api/test-executions/gettestexecutiondetails.mdx +++ b/src/pages/docs/api/test-executions/gettestexecutiondetails.mdx @@ -1,6 +1,6 @@ --- title: "Get test execution details" -description: "Retrieves a test execution with its call executions." +description: "Retrieves a test execution with paginated call executions and column configuration." --- - Filter by scenario name, transcript content, or status. + Filter call executions by phone number or scenario name. Page number. Defaults to `1`. + + Number of call executions per page. Defaults to `30`. + - JSON-encoded filter array, e.g. `[{"colId":"status","filterType":"text","type":"equals","filter":"completed"}]`. + JSON-encoded array of filter objects. Each object must contain a `column_id` and a `filter_config` object. + + **Structure:** + ```json + [ + { + "column_id": "", + "filter_config": { + "filter_type": "", + "filter_op": "", + "filter_value": "" + } + } + ] + ``` + + **`column_id` values:** `status`, `timestamp`, `call_execution_id`, `overall_score`, `response_time`, `call_type`, `scenario`, or an eval config UUID. + + **`filter_type` values:** `text`, `number`, `datetime`, `boolean`, `list`. + + **`filter_op` values:** `equals`, `not_equals`, `contains`, `not_contains`, `greater_than`, `less_than`, `greater_than_or_equal`, `less_than_or_equal`, `between`, `not_in_between`, `in`. + + **`filter_value`:** A string, number, ISO 8601 datetime string, or array (for `between` / `in` operators). + + **Example — filter by status:** + ```json + [{"column_id":"status","filter_config":{"filter_type":"text","filter_op":"equals","filter_value":"completed"}}] + ``` + + **Example — filter by score range:** + ```json + [{"column_id":"overall_score","filter_config":{"filter_type":"number","filter_op":"between","filter_value":[50,90]}}] + ``` JSON-encoded array of column IDs to group by, e.g. `["scenario"]`. @@ -79,23 +121,13 @@ description: "Retrieves a test execution with its call executions." - UUID of the test execution. - UUID of the parent run test. - Parent run test name. - Agent definition name. - Status: `pending`, `running`, `completed`, `failed`, `cancelled`, `cancelling`, or `evaluating`. - Failure reason. - ISO 8601 execution start time. - ISO 8601 completion time. - Number of distinct scenarios. - Total call executions created. - Calls that completed. - Calls that failed. - Execution metadata. - Elapsed time in seconds. - Percentage of calls completed successfully. - Paginated call execution objects. - + Total number of call executions. + URL for the next page, or `null` if on the last page. + URL for the previous page, or `null` if on the first page. + Total number of pages. + Current page number. + Paginated list of call execution objects. + UUID of the call execution. Call status: `pending`, `queued`, `ongoing`, `completed`, `failed`, `analyzing`, or `cancelled`. Duration in seconds. @@ -105,18 +137,25 @@ description: "Retrieves a test execution with its call executions." Scenario name. ISO 8601 creation timestamp. - ISO 8601 creation timestamp. - Scenario UUIDs included. - Simulator agent name. - UUID of the simulator agent. - Agent definition name used. - UUID of the agent definition used. - Total calls initiated. - Percentage of calls that connected. + Column configuration for the test execution grid. + + UUID of the column. + Display name of the column. + Whether the column is visible. + Data type of the column. + `scenario_dataset_column`, `evaluation`, or `tool_evaluation`. + UUID of the associated scenario, if applicable. + UUID of the associated dataset. + Eval configuration details, if applicable. + + Test execution status: `pending`, `running`, `completed`, `failed`, `cancelled`, `cancelling`, or `evaluating`. + List of error message strings, if any. + Agent provider name (e.g. `vapi`, `prompt`). + `voice` or `text`. Invalid or missing credentials. - Test execution not found. + Test execution not found or organization not found. Unexpected server error. diff --git a/src/pages/docs/api/test-executions/reruncalls.mdx b/src/pages/docs/api/test-executions/reruncalls.mdx index 8c0a8ed2..5e79db3c 100644 --- a/src/pages/docs/api/test-executions/reruncalls.mdx +++ b/src/pages/docs/api/test-executions/reruncalls.mdx @@ -10,16 +10,21 @@ description: "Reruns call executions within a test execution." parameters={[ {"name": "test_execution_id", "in": "path", "required": true, "description": "UUID of the test execution.", "type": "string"} ]} - requestBody={{"rerunType": "eval_only", "callExecutionIds": ["a1b2c3d4-e5f6-7890-abcd-ef1234567890"], "selectAll": false}} + requestBody={{"rerun_type": "eval_only", "call_execution_ids": ["a1b2c3d4-e5f6-7890-abcd-ef1234567890"], "select_all": false}} responseExample={{ message: "Rerun initiated successfully", - testExecutionId: "f7a8b9c0-d1e2-3456-789a-bcdef0123456", - rerunType: "eval_only", - totalProcessed: 1, - successfulReruns: ["a1b2c3d4-e5f6-7890-abcd-ef1234567890"], - failedReruns: [], - successCount: 1, - failureCount: 0 + test_execution_id: "f7a8b9c0-d1e2-3456-789a-bcdef0123456", + rerun_type: "eval_only", + total_processed: 2, + successful_reruns: ["a1b2c3d4-e5f6-7890-abcd-ef1234567890"], + failed_reruns: [ + { + call_execution_id: "b2c3d4e5-f6a7-8901-bcde-f01234567891", + error: "Call execution is in an incompatible state for rerun" + } + ], + success_count: 1, + failure_count: 1 }} responseStatus={200} responseStatusText="OK" @@ -41,14 +46,14 @@ description: "Reruns call executions within a test execution." - + The type of rerun to perform. Use `eval_only` to re-evaluate existing call data without re-executing the actual calls -- this is useful when you have updated your evaluation configurations and want to see updated scores without the cost of re-running calls. Use `call_and_eval` to fully re-execute the calls and then evaluate the new results -- this produces fresh conversations and is useful when you have modified the agent under test. Note that text agents only support `eval_only` reruns; attempting `call_and_eval` on a text agent will return a 400 error. - - An array of call execution UUIDs to rerun. Required when `selectAll` is `false` or not provided. Each ID must correspond to a valid call execution within the specified test execution. If a provided ID does not exist or does not belong to the test execution, it will appear in the `failedReruns` array of the response. + + An array of call execution UUIDs to rerun. Required when `select_all` is `false` or not provided. Each ID must correspond to a valid call execution within the specified test execution. If a provided ID does not exist or does not belong to the test execution, it will appear in the `failed_reruns` array of the response. - - When set to `true`, all call executions within the test execution will be rerun, and the `callExecutionIds` field is ignored. Defaults to `false`. You must provide either `selectAll: true` or a non-empty `callExecutionIds` array -- the request will fail with a 400 error if neither is specified. + + When set to `true`, all call executions within the test execution will be rerun, and the `call_execution_ids` field is ignored. Defaults to `false`. You must provide either `select_all: true` or a non-empty `call_execution_ids` array -- the request will fail with a 400 error if neither is specified. @@ -56,32 +61,32 @@ description: "Reruns call executions within a test execution." A human-readable confirmation message indicating that the rerun has been initiated. The actual rerun processing happens asynchronously after this response is returned. - + The UUID of the test execution that the rerun was initiated for, echoed back for confirmation and reference. - + The type of rerun that was requested, either `eval_only` or `call_and_eval`. Echoed back from the request for confirmation. - + The total number of call executions that were processed by the rerun request. This includes both successful and failed reruns. - + An array of call execution UUIDs that were successfully queued for rerun. These calls will be re-executed or re-evaluated asynchronously. - - An array of objects describing call executions that could not be rerun. Each object contains a `callExecutionId` (the UUID of the failed call) and an `error` (a human-readable description of why the rerun failed, such as the call being in an incompatible state). + + An array of objects describing call executions that could not be rerun. Each object contains a `call_execution_id` (the UUID of the failed call) and an `error` (a human-readable description of why the rerun failed, such as the call being in an incompatible state). - - The number of call executions that were successfully queued for rerun. Equal to the length of the `successfulReruns` array. + + The number of call executions that were successfully queued for rerun. Equal to the length of the `successful_reruns` array. - - The number of call executions that failed to be queued for rerun. Equal to the length of the `failedReruns` array. + + The number of call executions that failed to be queued for rerun. Equal to the length of the `failed_reruns` array. - The rerun request could not be processed. This error occurs when: the `rerunType` field is missing or contains an invalid value; neither `callExecutionIds` nor `selectAll` was provided; the test execution is still in an active state (`pending`, `running`, or `cancelling`) and cannot accept reruns; or a `call_and_eval` rerun was requested for a text agent, which only supports `eval_only` reruns. Check the error message in the response body for specific details on which validation failed. + The rerun request could not be processed. This error occurs when: the `rerun_type` field is missing or contains an invalid value; neither `call_execution_ids` nor `select_all` was provided; the test execution is still in an active state (`pending`, `running`, or `cancelling`) and cannot accept reruns; or a `call_and_eval` rerun was requested for a text agent, which only supports `eval_only` reruns. Check the error message in the response body for specific details on which validation failed. The request could not be authenticated. Verify that both `X-Api-Key` and `X-Secret-Key` headers are present and contain valid, non-expired credentials. Ensure the API key has access to the workspace that owns this test execution.