From 59e81dd6d0ce966b557ae31766e31e194cf5a8a8 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Wed, 12 Aug 2026 17:40:47 +0000 Subject: [PATCH] feat(eval): add ondemand evaluate (synchronous, client-side) --- src/core/eval.tsx | 338 ++++++++++ src/core/types.tsx | 2 +- src/handlers/eval/index.tsx | 4 +- .../EvaluateCommand.76a9b708fd699093.json | 23 + .../EvaluateCommand.c0c383465d2d11bc.json | 23 + ...tAgentRuntimeCommand.9f77333d1b9dcf5d.json | 47 ++ .../GetEvaluatorCommand.716589b0884f35c0.json | 51 ++ ...tQueryResultsCommand.19164cb5cd9b9b70.json | 586 ++++++++++++++++++ ...tQueryResultsCommand.275f4a33f37e8fc7.json | 212 +++++++ .../StartQueryCommand.1623083efa971538.json | 3 + .../StartQueryCommand.8981323eb27080a.json | 3 + .../__fixtures__/evaluate.golden.json | 44 ++ src/handlers/eval/ondemand/evaluate/index.tsx | 135 ++++ src/handlers/eval/ondemand/index.tsx | 13 + .../eval/ondemand/ondemand.fixture.test.tsx | 88 +++ src/handlers/eval/ondemand/ondemand.test.tsx | 178 ++++++ src/handlers/eval/types.tsx | 60 ++ src/testing/TestCoreClient.tsx | 35 ++ 18 files changed, 1843 insertions(+), 2 deletions(-) create mode 100644 src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json create mode 100644 src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json create mode 100644 src/handlers/eval/ondemand/evaluate/index.tsx create mode 100644 src/handlers/eval/ondemand/index.tsx create mode 100644 src/handlers/eval/ondemand/ondemand.fixture.test.tsx create mode 100644 src/handlers/eval/ondemand/ondemand.test.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index c7e1baaa8..d83a9eed3 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -33,19 +33,32 @@ import { type DataSourceConfig, type ListOnlineEvaluationConfigsResponse, type Rule, + type EvaluatorLevel, type UpdateEvaluatorResponse, type UpdateOnlineEvaluationConfigResponse, type BedrockAgentCoreControlClient, } from "@aws-sdk/client-bedrock-agentcore-control"; import { + EvaluateCommand, GetBatchEvaluationCommand, ListBatchEvaluationsCommand, StartBatchEvaluationCommand, + type EvaluationReferenceInput, + type EvaluationResultContent, + type EvaluationTarget, type ListBatchEvaluationsResponse, type StartBatchEvaluationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, type CloudWatchFilterConfig, } from "@aws-sdk/client-bedrock-agentcore"; +import { + GetQueryResultsCommand, + ResourceNotFoundException, + StartQueryCommand, + type CloudWatchLogsClient, + type ResultField, +} from "@aws-sdk/client-cloudwatch-logs"; +import type { DocumentType } from "@smithy/types"; import { Transform } from "node:stream"; import { FileWriteError, InputValidationError, NetworkingError } from "../errors"; import type { @@ -55,9 +68,14 @@ import type { CoreEvalClient, CreateDatasetInput, CreateOnlineEvalInput, + EvaluateInput, + EvaluateResult, GetBatchEvaluationResult, + GetTracesInput, LlmAsAJudgeUpdate, SessionSourceValue, + SessionTrace, + SpanRecord, StartBatchEvaluationInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -77,6 +95,19 @@ import { const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; +// The shared, account-level OTel span log group. +const SPANS_LOG_GROUP = "aws/spans"; + +// Default discovery window when no explicit --start/--end or --lookback-days is +// given. Mirrors the batch service's now-7d default. +const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000; + +// A hard Evaluate limit: at most 10 trace/span ids per request. +const EVALUATE_TARGET_BATCH = 10; + +// CloudWatch Logs Insights hard ceiling: a query returns at most 100k rows. +const INSIGHTS_MAX_ROWS = 100_000; + // noopLogger is the default for the optional logger arg so callers that don't // need batch-evaluation result-log diagnostics (e.g. dataset-only tests) can // omit it. Production (src/core/index.tsx) injects a real child logger. @@ -339,6 +370,98 @@ export class EvalClient implements CoreEvalClient { }; } + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { + const qualifier = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; + + const { runtimeId, runtimeName } = await resolveAgentToNameAndId( + input.agent, + this.clients, + options, + ); + const logGroupName = runtimeLogGroup(runtimeId, qualifier); + const serviceName = runtimeServiceName(runtimeName, qualifier); + + // CloudWatch Insights takes epoch seconds. Discovery defaults to now-7d when + // no explicit window is given (matches the batch service's default). + const endMs = input.window ? +input.window.endTime : Date.now(); + const startMs = input.window ? +input.window.startTime : endMs - SEVEN_DAYS_MS; + const startSec = Math.floor(startMs / 1000); + const endSec = Math.floor(endMs / 1000); + + const logs = this.clients.logs(toClientConfig(options)); + const queryString = buildSpanQuery(serviceName, input.sessionIds, input.traceId); + + // Runtime group required (missing = agent has no traces); aws/spans optional now + // https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html#observability-configure-unified-traces + const [runtimeRows, sharedRows] = await Promise.all([ + runInsightsQuery(logs, [logGroupName], queryString, startSec, endSec).catch((error) => { + if (error instanceof ResourceNotFoundException) { + throw new InputValidationError( + `No telemetry found for agent "${input.agent}": its runtime log group ${logGroupName} ` + + `does not exist. Ensure the agent has been invoked and emits traces.`, + { meta: { agent: input.agent, logGroupName } }, + ); + } + throw error; + }), + runInsightsQuery(logs, [SPANS_LOG_GROUP], queryString, startSec, endSec).catch((error) => { + if (error instanceof ResourceNotFoundException) return []; + throw error; + }), + ]); + const traces = groupSpansBySession([...sharedRows, ...runtimeRows]); + + // Warn when explicitly requested sessions never showed up in the logs (aged + // out, wrong id, or never emitted) so a caller isn't misled by a partial run. + if (input.sessionIds?.length) { + const found = new Set(traces.map((t) => t.sessionId)); + const missing = input.sessionIds.filter((id) => !found.has(id)); + if (missing.length > 0) { + this.logger.warn(`requested sessions not found in logs: ${missing.join(", ")}`); + } + } + return traces; + } + + async evaluate(input: EvaluateInput, options: CoreOptions): Promise { + const data = this.clients.data(toClientConfig(options)); + const control = this.clients.control(toClientConfig(options)); + const levels = await resolveEvaluatorLevels(input.evaluatorIds, control, options); + const refsBySession = groupRefsBySession(input.groundTruth); + + const results: EvaluationResultContent[] = []; + // Sessions that actually produced Evaluate results — distinct from the sessions + // handed in, since a TRACE/TOOL_CALL session with no matching ids makes no call. + const evaluatedSessions = new Set(); + for (const evaluatorId of input.evaluatorIds) { + const level = levels.get(evaluatorId) ?? "SESSION"; + for (const trace of input.traces) { + // TRACE/TOOL_CALL sessions with no ids at that level contribute no calls + // (empty batch list); SESSION always makes one call with no target. + for (const target of targetBatches(level, trace)) { + const response = await data.send( + new EvaluateCommand({ + evaluatorId, + // SpanRecord is Record for ergonomic reads; the API + // wants DocumentType[] (assignable to it, so a single cast suffices). + evaluationInput: { sessionSpans: trace.spans as DocumentType[] }, + evaluationTarget: target, + evaluationReferenceInputs: refsBySession.get(trace.sessionId), + }), + ); + const evaluationResults = response.evaluationResults ?? []; + if (evaluationResults.length > 0) evaluatedSessions.add(trace.sessionId); + results.push(...evaluationResults); + } + } + } + return { + sessionsRequested: input.traces.length, + sessionsEvaluated: evaluatedSessions.size, + results, + }; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, @@ -787,6 +910,221 @@ async function agentDataSource( }; } +// sanitizeQueryValue strips single quotes so an id can't break out of the quoted +// Insights filter literal it is interpolated into (matches the old CLI). +function sanitizeQueryValue(value: string): string { + return value.replace(/'/g, ""); +} + +// buildSpanQuery is the single-phase Insights query: scope to one runtime by its +// OTel service.name, optionally narrow to specific sessions and/or one trace, and +// select the full span JSON (@message) plus the session id to group by. It does +// NOT over-filter on ispresent(kind) — that span-only predicate is what forced the +// old CLI's second query for log records; the looser scope returns everything for +// the session in one pass. +function buildSpanQuery(serviceName: string, sessionIds?: string[], traceId?: string): string { + let query = `fields @message, attributes.session.id as sessionId, traceId, spanId + | filter resource.attributes.service.name in ['${sanitizeQueryValue(serviceName)}']`; + if (sessionIds && sessionIds.length > 0) { + const ids = sessionIds.map((id) => `'${sanitizeQueryValue(id)}'`).join(", "); + query += `\n | filter attributes.session.id in [${ids}]`; + } + if (traceId) { + query += `\n | filter traceId = '${sanitizeQueryValue(traceId)}'`; + } + query += `\n | sort @timestamp asc\n | limit ${INSIGHTS_MAX_ROWS}`; + return query; +} + +// runInsightsQuery starts a CloudWatch Logs Insights query, waits for it to finish, +// then drains all result pages. GetQueryResults returns <=10k rows per call, so a +// long session's spans span multiple pages (nextToken); dropping any would score a +// partial conversation. Fails fast if the ceiling is hit — that belongs in batch. +async function runInsightsQuery( + logs: CloudWatchLogsClient, + logGroupNames: string[], + queryString: string, + startSec: number, + endSec: number, +): Promise { + const started = await logs.send( + new StartQueryCommand({ logGroupNames, queryString, startTime: startSec, endTime: endSec }), + ); + const queryId = started.queryId; + + // Phase 1: wait for completion. A large scan can take minutes, so the deadline is + // generous; each poll costs one cheap GetQueryResults call. + let status = "Running"; + for (let i = 0; i < 300 && status !== "Complete"; i++) { + const result = await logs.send(new GetQueryResultsCommand({ queryId })); + status = result.status ?? "Unknown"; + if (status === "Failed" || status === "Cancelled" || status === "Timeout") { + throw new NetworkingError(`CloudWatch Logs Insights query ${status.toLowerCase()}`, { + meta: { queryId }, + }); + } + if (status !== "Complete") await new Promise((resolve) => setTimeout(resolve, 1000)); + } + if (status !== "Complete") { + throw new NetworkingError("CloudWatch Logs Insights query did not finish in time", { + meta: { queryId }, + }); + } + + // Phase 2: drain pages. Terminates on nextToken; total is bounded by the query's + // `| limit INSIGHTS_MAX_ROWS`. + const rows: ResultField[][] = []; + let nextToken: string | undefined; + do { + const result = await logs.send(new GetQueryResultsCommand({ queryId, nextToken })); + rows.push(...(result.results ?? [])); + nextToken = result.nextToken; + } while (nextToken); + + if (rows.length >= INSIGHTS_MAX_ROWS) { + throw new InputValidationError( + `Too many spans in scope (>= ${INSIGHTS_MAX_ROWS}). Narrow --session-ids or the time ` + + `window, or use 'eval batch-evaluation' for large jobs.`, + ); + } + return rows; +} + +// Group parsed @message docs by session, keeping only sessions with >=1 span +// (Evaluate rejects log-only sessions), and derive each session's trace/tool ids. +function groupSpansBySession(rows: ResultField[][]): SessionTrace[] { + const docsBySession = new Map(); + const sessionsWithSpans = new Set(); + for (const row of rows) { + const message = row.find((f) => f.field === "@message")?.value; + const sessionId = row.find((f) => f.field === "sessionId")?.value; + // Drop orphan records with no session id (e.g. system logs) — an + // unidentifiable session can't be evaluated. + if (!message || !sessionId) continue; + let doc: SpanRecord; + try { + doc = JSON.parse(message) as SpanRecord; + } catch { + continue; + } + const list = docsBySession.get(sessionId); + if (list) list.push(doc); + else docsBySession.set(sessionId, [doc]); + // distinquish between log record and span (which has "kind") + if ("kind" in doc) sessionsWithSpans.add(sessionId); + } + return [...docsBySession] + .filter(([sessionId]) => sessionsWithSpans.has(sessionId)) + .map(([sessionId, spans]) => ({ + sessionId, + spans, + traceIds: extractTraceIds(spans), + toolCallSpanIds: extractToolCallSpanIds(spans), + })); +} + +// extractTraceIds pulls the distinct trace ids out of a session's spans, preserving +// first-seen order (ported from the old CLI's span-collector). +function extractTraceIds(spans: SpanRecord[]): string[] { + const seen = new Set(); + const traceIds: string[] = []; + for (const span of spans) { + const traceId = span.traceId; + // Skip empty ids: log records not tied to a trace carry traceId "", and the + // Evaluate API rejects any target id that isn't a 32-char trace id. + if (typeof traceId === "string" && traceId.length > 0 && !seen.has(traceId)) { + seen.add(traceId); + traceIds.push(traceId); + } + } + return traceIds; +} + +// isToolSpan classifies a tool-execution span by the framework markers the AgentCore +// evaluation SDK uses (Strands / OpenInference / Traceloop). Exact, case-sensitive — +// OpenInference is "TOOL" (upper), Traceloop is "tool" (lower). +function isToolSpan(attrs: Record): boolean { + return ( + attrs["gen_ai.operation.name"] === "execute_tool" || + attrs["openinference.span.kind"] === "TOOL" || + attrs["traceloop.span.kind"] === "tool" + ); +} + +// extractToolCallSpanIds pulls the span ids of tool-execution spans for TOOL_CALL +// evaluators. A tool name attribute alone is unreliable (Traceloop names tools via +// traceloop.entity.name, not tool.name), so classify by span kind instead. +function extractToolCallSpanIds(spans: SpanRecord[]): string[] { + const spanIds: string[] = []; + for (const span of spans) { + const spanId = span.spanId; + if (typeof spanId !== "string" || spanId.length === 0) continue; + if (isToolSpan((span.attributes ?? {}) as Record)) spanIds.push(spanId); + } + return spanIds; +} + +// resolveEvaluatorLevels maps each evaluator id to its evaluation level via +// GetEvaluator — authoritative for both builtins (e.g. Builtin.Helpfulness ⇒ TRACE) +// and custom evaluators, so the level (which decides whether the Evaluate call +// targets trace ids, tool-call span ids, or the whole session) is never guessed. +// GetEvaluator errors propagate: a failed lookup (e.g. AccessDenied, not-found) +// must surface, not silently degrade to SESSION and submit the wrong scope. +async function resolveEvaluatorLevels( + evaluatorIds: string[], + control: BedrockAgentCoreControlClient, + _options: CoreOptions, +): Promise> { + const levels = new Map(); + for (const id of new Set(evaluatorIds)) { + const evaluator = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + // level is a required response field (SDK types it `| undefined` without `?`). + levels.set(id, evaluator.level!); + } + return levels; +} + +// groupRefsBySession indexes ground-truth reference inputs by the session they +// apply to (context.spanContext.sessionId), so each Evaluate call attaches only its +// own session's references — the synchronous Evaluate shape, not batch's job-level +// evaluationMetadata. +function groupRefsBySession( + groundTruth: EvaluationReferenceInput[] | undefined, +): Map { + const map = new Map(); + for (const ref of groundTruth ?? []) { + const sessionId = + ref.context && "spanContext" in ref.context ? ref.context.spanContext?.sessionId : undefined; + if (!sessionId) continue; + const list = map.get(sessionId); + if (list) list.push(ref); + else map.set(sessionId, [ref]); + } + return map; +} + +// targetBatches splits a session's evaluation targets into <=10-id Evaluate calls. +// SESSION evaluators make a single call with no target; TRACE/TOOL_CALL sessions +// with no ids at that level make none (the session is skipped for that evaluator). +function targetBatches( + level: EvaluatorLevel, + trace: SessionTrace, +): (EvaluationTarget | undefined)[] { + if (level === "TRACE") { + return chunk(trace.traceIds, EVALUATE_TARGET_BATCH).map((traceIds) => ({ traceIds })); + } + if (level === "TOOL_CALL") { + return chunk(trace.toolCallSpanIds, EVALUATE_TARGET_BATCH).map((spanIds) => ({ spanIds })); + } + return [undefined]; +} + +function chunk(items: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)); + return out; +} + // A just-written role or inline policy is not visible to the service immediately // (IAM is eventually consistent), and the service validates both when the config // is created. It surfaces as one of two messages depending on which part has not diff --git a/src/core/types.tsx b/src/core/types.tsx index 98a2f338d..9b1a2d4a8 100644 --- a/src/core/types.tsx +++ b/src/core/types.tsx @@ -37,7 +37,7 @@ export type CoreFetch = ( // full ClientConfig so callers can request any client customization (region, // endpoint, ...). export interface AwsClients { - control(config: ClientConfig): BedrockAgentCoreControlClient; + control(config: ClientConfig): BedrockAgentCoreControlClient data(config: ClientConfig): BedrockAgentCoreClient; iam(config: ClientConfig): IAMClient; // logs reads the CloudWatch Logs streams AgentCore writes batch-evaluation diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index da3e6c686..0ecef2055 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -7,6 +7,7 @@ import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; import { createDatasetHandler } from "./dataset"; import { createBatchEvaluationHandler } from "./batch-evaluation"; +import { createOnDemandHandler } from "./ondemand"; export function createEvalHandler(core: Core, io: AppIO): Router { return new Router("eval", "evaluate and optimize AgentCore agents") @@ -15,7 +16,8 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createEvaluatorHandler(core, io)) .handler(createOnlineEvalHandler(core, io)) .handler(createDatasetHandler(core, io)) - .handler(createBatchEvaluationHandler(core, io)); + .handler(createBatchEvaluationHandler(core, io)) + .handler(createOnDemandHandler(core, io)); } export { EvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json new file mode 100644 index 000000000..31796ea2d --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json @@ -0,0 +1,23 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a", + "traceId": "6a7cabfa3bfe9a7348415c7b507648a8" + } + }, + "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 941, + "outputTokens": 118, + "totalTokens": 1059 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json new file mode 100644 index 000000000..f0b13e303 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json @@ -0,0 +1,23 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", + "traceId": "6a7cac0e3fa42bfe5437eef070d1231c" + } + }, + "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.", + "value": 1, + "label": "Above And Beyond", + "tokenUsage": { + "inputTokens": 859, + "outputTokens": 154, + "totalTokens": 1013 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json new file mode 100644 index 000000000..2d3b5e713 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json @@ -0,0 +1,47 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeName": "asdf_MyAgent", + "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeVersion": "1", + "createdAt": { + "$date": "2026-04-23T21:17:21.895Z" + }, + "lastUpdatedAt": { + "$date": "2026-04-23T21:17:35.159Z" + }, + "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: asdf_MyAgent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2", + "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip" + } + }, + "runtime": "PYTHON_3_13", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE", + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json new file mode 100644 index 000000000..1825df228 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json @@ -0,0 +1,51 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorConfig": { + "llmAsAJudge": { + "ratingScale": { + "numerical": [ + { + "value": 0, + "label": "Not helpful at all" + }, + { + "value": 1, + "label": "Very unhelpful" + }, + { + "value": 2, + "label": "Somewhat unhelpful" + }, + { + "value": 3, + "label": "Neutral/Mixed" + }, + { + "value": 4, + "label": "Somewhat helpful" + }, + { + "value": 5, + "label": "Very helpful" + }, + { + "value": 6, + "label": "Above and beyond" + } + ] + } + } + }, + "level": "TRACE", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "lockedForModification": true +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json new file mode 100644 index 000000000..f5626155b --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json @@ -0,0 +1,586 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"1d3bcfe3653d81e4\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392103874218,\"endTimeUnixNano\":1786555392105289657,\"durationNano\":1415439,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "1d3bcfe3653d81e4" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"11655ee4cdde1872\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392105841503,\"endTimeUnixNano\":1786555392106502618,\"durationNano\":661115,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "11655ee4cdde1872" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"210fa6dea654dcbc\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392106868305,\"endTimeUnixNano\":1786555392107751277,\"durationNano\":882972,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "210fa6dea654dcbc" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"cde6ff0cb3289288\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392241890106,\"endTimeUnixNano\":1786555392285594042,\"durationNano\":43703936,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "cde6ff0cb3289288" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"9d3e945db9599945\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392289575652,\"endTimeUnixNano\":1786555392316954181,\"durationNano\":27378529,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "9d3e945db9599945" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"31efc7027314ac76\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392318607342,\"endTimeUnixNano\":1786555392369734375,\"durationNano\":51127033,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "31efc7027314ac76" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"53a0cd3ac657fb83\",\"parentSpanId\":\"db6cce231beea7f0\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392389258299,\"endTimeUnixNano\":1786555394009438471,\"durationNano\":1620180172,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"c1b5339e-9b84-45b0-a4ad-cfd818de8fca\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"tool_use\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1210,\"aws.genai.token_count_total\":1280,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH3RTZMMT4\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "53a0cd3ac657fb83" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"db6cce231beea7f0\",\"parentSpanId\":\"7f719fd629b95bca\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392374099914,\"endTimeUnixNano\":1786555394010008258,\"durationNano\":1635908344,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1210,\"gen_ai.usage.output_tokens\":70,\"gen_ai.server.request.duration\":1601,\"gen_ai.usage.total_tokens\":1280,\"gen_ai.usage.completion_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.374107+00:00\",\"gen_ai.server.time_to_first_token\":1613,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.009969+00:00\",\"gen_ai.usage.input_tokens\":1210,\"aws.genai.token_count_total\":1280,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "db6cce231beea7f0" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"26952cbd9a43fe35\",\"parentSpanId\":\"7f719fd629b95bca\",\"flags\":256,\"name\":\"execute_tool add_numbers\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394010521473,\"endTimeUnixNano\":1786555394011315403,\"durationNano\":793930,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.tool.status\":\"success\",\"gen_ai.tool.call.id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\",\"gen_ai.tool.description\":\"Return the sum of two numbers\",\"gen_ai.tool.json_schema\":\"{\\\"properties\\\": {\\\"a\\\": {\\\"description\\\": \\\"Parameter a\\\", \\\"type\\\": \\\"integer\\\"}, \\\"b\\\": {\\\"description\\\": \\\"Parameter b\\\", \\\"type\\\": \\\"integer\\\"}}, \\\"required\\\": [\\\"a\\\", \\\"b\\\"], \\\"type\\\": \\\"object\\\"}\",\"aws.genai.span_kind\":\"TOOL\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.010538+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"execute_tool\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.011292+00:00\",\"gen_ai.tool.name\":\"add_numbers\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "26952cbd9a43fe35" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7f719fd629b95bca\",\"parentSpanId\":\"a5e83c7e772efdf6\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392373929874,\"endTimeUnixNano\":1786555394011480146,\"durationNano\":1637550272,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.011468+00:00\",\"event_loop.cycle_id\":\"23f63c39-82fe-4159-8035-ff8e6208b88f\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.373944+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7f719fd629b95bca" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"0abec106a8ff39b7\",\"parentSpanId\":\"7576c2384130fece\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555394012454655,\"endTimeUnixNano\":1786555397051836921,\"durationNano\":3039382266,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"8926bd3b-32aa-42a7-a234-5bafc1cd8537\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1294,\"aws.genai.token_count_total\":1306,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH3RTZMMT4\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "0abec106a8ff39b7" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7576c2384130fece\",\"parentSpanId\":\"a1d0f9231c5db33d\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394011912861,\"endTimeUnixNano\":1786555397052514047,\"durationNano\":3040601186,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1294,\"gen_ai.usage.output_tokens\":12,\"gen_ai.server.request.duration\":3034,\"gen_ai.usage.total_tokens\":1306,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.011919+00:00\",\"gen_ai.server.time_to_first_token\":2804,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.052476+00:00\",\"gen_ai.usage.input_tokens\":1294,\"aws.genai.token_count_total\":1306,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7576c2384130fece" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a1d0f9231c5db33d\",\"parentSpanId\":\"a5e83c7e772efdf6\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394011746512,\"endTimeUnixNano\":1786555397052873967,\"durationNano\":3041127455,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.052853+00:00\",\"event_loop.cycle_id\":\"beebe4f0-27a6-4aef-a0e3-09cda0c61a19\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.011757+00:00\",\"event_loop.parent_cycle_id\":\"23f63c39-82fe-4159-8035-ff8e6208b88f\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a1d0f9231c5db33d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"5aa1ead48a6ac0e1\",\"parentSpanId\":\"e1355f7dda87635b\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1786555392052368325,\"endTimeUnixNano\":1786555397053469078,\"durationNano\":5001100753,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":55860,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "5aa1ead48a6ac0e1" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a5e83c7e772efdf6\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392372920661,\"endTimeUnixNano\":1786555397053072269,\"durationNano\":4680151608,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2504,\"gen_ai.usage.output_tokens\":82,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2586,\"gen_ai.usage.completion_tokens\":82,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.372942+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.053049+00:00\",\"gen_ai.usage.input_tokens\":2504,\"aws.genai.token_count_total\":2586,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a5e83c7e772efdf6" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"42f4da466ab4c999\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411324466077,\"endTimeUnixNano\":1786555411324998705,\"durationNano\":532628,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "42f4da466ab4c999" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"6d003c695538a610\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411323051352,\"endTimeUnixNano\":1786555411324071465,\"durationNano\":1020113,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "6d003c695538a610" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"8a05cdd0a93d3a6a\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411325315616,\"endTimeUnixNano\":1786555411325825454,\"durationNano\":509838,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "8a05cdd0a93d3a6a" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"0745de30c628c8e5\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411431858312,\"endTimeUnixNano\":1786555411485160682,\"durationNano\":53302370,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "0745de30c628c8e5" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"cb5ae6a1e269e21b\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411487794771,\"endTimeUnixNano\":1786555411518235079,\"durationNano\":30440308,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "cb5ae6a1e269e21b" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"b043a295f3c1c4cf\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411519590941,\"endTimeUnixNano\":1786555411598192831,\"durationNano\":78601890,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "b043a295f3c1c4cf" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"b556c691098ede16\",\"parentSpanId\":\"627f6364bb0599fa\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411614408678,\"endTimeUnixNano\":1786555414476372175,\"durationNano\":2861963497,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"47288f71-5ae9-4b15-9720-d49cf3e180f2\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":48,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHYLSAHMVB\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "b556c691098ede16" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"627f6364bb0599fa\",\"parentSpanId\":\"e90c8e0cb4712af5\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411601581872,\"endTimeUnixNano\":1786555414476887232,\"durationNano\":2875305360,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":48,\"gen_ai.server.request.duration\":2842,\"gen_ai.usage.total_tokens\":1251,\"gen_ai.usage.completion_tokens\":48,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.601589+00:00\",\"gen_ai.server.time_to_first_token\":1966,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.476851+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "627f6364bb0599fa" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"e90c8e0cb4712af5\",\"parentSpanId\":\"00821fcc8a234519\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411601446390,\"endTimeUnixNano\":1786555414477214324,\"durationNano\":2875767934,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.477197+00:00\",\"event_loop.cycle_id\":\"cbaf6d7e-50f0-436e-8c06-6c5d9e6c5936\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.601458+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "e90c8e0cb4712af5" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA0YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"00821fcc8a234519\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411600856280,\"endTimeUnixNano\":1786555414477366577,\"durationNano\":2876510297,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":48,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1251,\"gen_ai.usage.completion_tokens\":48,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.600874+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.477345+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "00821fcc8a234519" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA4YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"82efc476a79eaee2\",\"parentSpanId\":\"1cc0af0f799f67f1\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1786555411282407024,\"endTimeUnixNano\":1786555414477671431,\"durationNano\":3195264407,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":34652,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "82efc476a79eaee2" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA8YAQ==" + } + ] + ], + "statistics": { + "recordsMatched": 26, + "recordsScanned": 26, + "estimatedRecordsSkipped": 0, + "bytesScanned": 47213, + "estimatedBytesSkipped": 0, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json new file mode 100644 index 000000000..a4401cbb3 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json @@ -0,0 +1,212 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394010008258,\"observedTimeUnixNano\":1786555395351842794,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"tool_use\",\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"db6cce231beea7f0\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "db6cce231beea7f0" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394011480146,\"observedTimeUnixNano\":1786555395352095155,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\",\"tool.result\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}},{\"role\":\"assistant\",\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7f719fd629b95bca\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7f719fd629b95bca" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394011315403,\"observedTimeUnixNano\":1786555395351997309,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"tool\",\"content\":{\"role\":\"tool\",\"content\":\"{\\\"a\\\": 2, \\\"b\\\": 2}\",\"id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"text\\\": \\\"4\\\"}]\",\"id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"26952cbd9a43fe35\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "26952cbd9a43fe35" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397052514047,\"observedTimeUnixNano\":1786555400454612062,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"2 + 2 = 4\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7576c2384130fece\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7576c2384130fece" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397052873967,\"observedTimeUnixNano\":1786555400454777045,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a1d0f9231c5db33d\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a1d0f9231c5db33d" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397053072269,\"observedTimeUnixNano\":1786555400454894614,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"2 + 2 = 4\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a5e83c7e772efdf6\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a5e83c7e772efdf6" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414476887232,\"observedTimeUnixNano\":1786555419396909949,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"A primary color is **red**.\\\\n\\\\nThe three primary colors are red, blue, and yellow (in traditional color theory) or red, green, and blue (in the RGB/additive color model used for light).\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"627f6364bb0599fa\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "627f6364bb0599fa" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414477366577,\"observedTimeUnixNano\":1786555419397161059,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"A primary color is **red**.\\n\\nThe three primary colors are red, blue, and yellow (in traditional color theory) or red, green, and blue (in the RGB/additive color model used for light).\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"00821fcc8a234519\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "00821fcc8a234519" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414477214324,\"observedTimeUnixNano\":1786555419397054030,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"e90c8e0cb4712af5\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "e90c8e0cb4712af5" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQAxgB" + } + ] + ], + "statistics": { + "recordsMatched": 9, + "recordsScanned": 72, + "estimatedRecordsSkipped": 3, + "bytesScanned": 68886, + "estimatedBytesSkipped": 349, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json new file mode 100644 index 000000000..4320f369e --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json @@ -0,0 +1,3 @@ +{ + "queryId": "3c802de9-ea3d-4268-ae53-944d121d2618" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json new file mode 100644 index 000000000..cd9387e26 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json @@ -0,0 +1,3 @@ +{ + "queryId": "cdc73753-869e-4222-b606-a7e5b2e7d9a5" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json b/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json new file mode 100644 index 000000000..458c13bd7 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json @@ -0,0 +1,44 @@ +{ + "sessionsRequested": 2, + "sessionsEvaluated": 2, + "results": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a", + "traceId": "6a7cabfa3bfe9a7348415c7b507648a8" + } + }, + "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 941, + "outputTokens": 118, + "totalTokens": 1059 + } + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", + "traceId": "6a7cac0e3fa42bfe5437eef070d1231c" + } + }, + "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.", + "value": 1, + "label": "Above And Beyond", + "tokenUsage": { + "inputTokens": 859, + "outputTokens": 154, + "totalTokens": 1013 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/evaluate/index.tsx b/src/handlers/eval/ondemand/evaluate/index.tsx new file mode 100644 index 000000000..b65416509 --- /dev/null +++ b/src/handlers/eval/ondemand/evaluate/index.tsx @@ -0,0 +1,135 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { coreOptsFromCtx, parseJsonArrayFlag } from "../../../utils"; +import type { SessionWindow } from "../../types"; + +export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) => + createHandler({ + name: "evaluate", + description: "evaluate existing sessions client-side (synchronous; prints scores)", + flags: [ + flag( + "agent", + "source: harness id or runtime id whose sessions to evaluate", + z.string().optional(), + ), + flag("endpoint", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag( + "lookback-days", + "time filter: evaluate sessions from the last N days", + z.number().optional(), + ), + flag( + "start-time", + "time filter: window start (ISO-8601, with --end-time)", + z.string().optional(), + ), + flag( + "end-time", + "time filter: window end (ISO-8601, with --start-time)", + z.string().optional(), + ), + flag("session-ids", "filter: specific session ids", z.array(z.string()).optional()), + flag( + "trace-id", + "filter: a single trace id (session id is read off the span)", + z.string().optional(), + ), + flag( + "ground-truth", + "ground truth (JSON EvaluationReferenceInput[]; inline, file://, or -)", + z.string().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["agent"]) { + throw new InputValidationError("on-demand requires '--agent'"); + } + if (!flags["evaluator"] || flags["evaluator"].length === 0) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + const window = resolveWindow(flags); + const sessionIds = flags["session-ids"]; + const traceId = flags["trace-id"]; + // On-demand reads sessions client-side, so it needs a concrete source — + // "evaluate everything" is not allowed. + if (!window && !sessionIds?.length && !traceId) { + throw new InputValidationError( + "specify a session source: --session-ids, --trace-id, --lookback-days, or --start-time/--end-time", + ); + } + + const opts = coreOptsFromCtx(ctx); + const traces = await core.eval.getTracesForAgent( + { agent: flags["agent"], endpoint: flags["endpoint"], window, sessionIds, traceId }, + opts, + ); + + // Ground truth is a typed SDK-shape passthrough (identical to batch's handler): + // resolve inline / file:// / -, then hand the array to core verbatim — core + // groups it by session. + const resolver = new SourceResolver({ stdin: io.stdin }); + const groundTruth = parseJsonArrayFlag( + "ground-truth", + await resolver.resolveText("ground-truth", flags["ground-truth"]), + ); + + const result = await core.eval.evaluate( + { traces, evaluatorIds: flags["evaluator"], groundTruth }, + opts, + ); + ctx.require(JsonRendererKey).renderJson(result); + }, + }); + +// resolveWindow validates on-demand's time filter: --lookback-days maps to +// [now - n days, now]; the explicit --start-time/--end-time pair must come together +// with start before end. On-demand owns this rather than reusing batch's resolver: +// batch has no --lookback-days and its window feeds a service-side data source, not +// a client-side Insights query. +function resolveWindow(flags: { + "lookback-days"?: number; + "start-time"?: string; + "end-time"?: string; +}): SessionWindow | undefined { + const lookback = flags["lookback-days"]; + const hasStart = flags["start-time"] !== undefined; + const hasEnd = flags["end-time"] !== undefined; + + if (lookback !== undefined) { + if (hasStart || hasEnd) { + throw new InputValidationError( + "--lookback-days cannot be combined with --start-time/--end-time", + ); + } + if (!Number.isFinite(lookback) || lookback <= 0) { + throw new InputValidationError("--lookback-days must be a positive number"); + } + const endTime = new Date(); + const startTime = new Date(+endTime - lookback * 24 * 60 * 60 * 1000); + return { startTime, endTime }; + } + + if (!hasStart && !hasEnd) return undefined; + if (!hasStart || !hasEnd) { + throw new InputValidationError("--start-time and --end-time must be provided together"); + } + const startTime = new Date(flags["start-time"]!); + const endTime = new Date(flags["end-time"]!); + if (Number.isNaN(+startTime) || Number.isNaN(+endTime)) { + throw new InputValidationError("--start-time and --end-time must be ISO-8601 timestamps"); + } + if (+startTime >= +endTime) { + throw new InputValidationError("--start-time must be before --end-time"); + } + return { startTime, endTime }; +} diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx new file mode 100644 index 000000000..d46cc38ea --- /dev/null +++ b/src/handlers/eval/ondemand/index.tsx @@ -0,0 +1,13 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createEvaluateOnDemandHandler } from "./evaluate"; + +// ondemand groups the synchronous, client-side evaluation commands. It has no TUI +// screen (unlike evaluator/online-eval), so a bare invocation prints help. +export function createOnDemandHandler(core: Core, io: AppIO): Router { + return new Router("ondemand", "evaluate existing sessions synchronously, client-side") + .default(createHelpDefault(io)) + .handler(createEvaluateOnDemandHandler(core, io)); +} diff --git a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx new file mode 100644 index 000000000..f34081c7d --- /dev/null +++ b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx @@ -0,0 +1,88 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with: RECORD=1 bun test src/handlers/eval/ondemand/ondemand.fixture.test.tsx +// +// This exercises the real seam end to end: parsing → handler → CoreClient → +// getTracesForAgent (GetAgentRuntime + CloudWatch Logs Insights StartQuery / +// GetQueryResults, read from aws/spans and the runtime group) → evaluate (the +// Evaluate data-plane API) → rendered scores. +// +// Determinism: the window is PINNED (not --lookback-days) so the StartQuery input — +// which embeds startTime/endTime epoch seconds — hashes to the same fixture on +// record and replay. --session-ids bounds the fetch to the two sessions recorded +// against the live agent below. +// +// Re-recording needs the agent to still exist AND those sessions' spans to still be +// within CloudWatch retention (they age out). If they've aged out, invoke the agent +// to create fresh sessions, then repoint FIXTURE_SESSION_IDS + the window at them. +const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; +const FIXTURE_SESSION_IDS = [ + "67ebf93b-65e3-4127-9e13-483b239f256a", + "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", +]; +const WINDOW_START = "2026-08-12T00:00:00Z"; +const WINDOW_END = "2026-08-13T00:00:00Z"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +describe("eval ondemand evaluate (fixture-backed)", () => { + test("evaluates the target sessions client-side and prints scores", async () => { + const stdout = await run([ + "eval", + "ondemand", + "evaluate", + "--agent", + FIXTURE_AGENT, + "--session-ids", + ...FIXTURE_SESSION_IDS, + "--start-time", + WINDOW_START, + "--end-time", + WINDOW_END, + "--evaluator", + "Builtin.Helpfulness", + ]); + + matchGolden(FIXTURES, "evaluate.golden.json", stdout); + const result = JSON.parse(stdout); + expect(result.sessionsEvaluated).toBeGreaterThan(0); + expect(result.results.length).toBeGreaterThan(0); + expect(result.results[0].evaluatorId).toBe("Builtin.Helpfulness"); + // Recording polls live CloudWatch Insights (1s between polls) and calls Evaluate + // per session, so it needs well over bun's 5s default; replay is instant. + }, 180_000); +}); diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx new file mode 100644 index 000000000..6d7173444 --- /dev/null +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -0,0 +1,178 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../index"; +import { + createSilentLogger, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../testing"; +import type { EvaluateResult, SessionTrace } from "../types"; + +// Command-flow tests for `eval ondemand evaluate`, driven through the real root +// handler against a TestCoreClient (no network). These cover the handler's +// orchestration (getTracesForAgent → evaluate), source-arm validation, and the +// local window resolution. The end-to-end SDK path (Insights + Evaluate) is proven +// by the golden fixture suite, which must be recorded against a live account. + +const TRACE: SessionTrace = { + sessionId: "s1", + spans: [{ traceId: "t1", spanId: "sp1" }], + traceIds: ["t1"], + toolCallSpanIds: [], +}; + +const RESULT: EvaluateResult = { + sessionsRequested: 1, + sessionsEvaluated: 1, + results: [{ evaluatorId: "Builtin.Helpfulness", value: 0.9 } as EvaluateResult["results"][number]], +}; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setGetTracesResponse([TRACE]).setEvaluateResponse(RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +const BASE = [ + "eval", + "ondemand", + "evaluate", + "--agent", + "a-1", + "--evaluator", + "Builtin.Helpfulness", +]; + +describe("eval ondemand command hierarchy", () => { + test("registers evaluate under eval → ondemand", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "ondemand"); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + }); +}); + +describe("eval ondemand evaluate validation", () => { + test("requires --agent", async () => { + await expect( + run([ + "eval", + "ondemand", + "evaluate", + "--evaluator", + "Builtin.Helpfulness", + "--session-ids", + "s1", + ]), + ).rejects.toThrow(/--agent/); + }); + + test("requires --evaluator", async () => { + await expect( + run(["eval", "ondemand", "evaluate", "--agent", "a-1", "--session-ids", "s1"]), + ).rejects.toThrow(/--evaluator/); + }); + + test("rejects an empty session source", async () => { + await expect(run(BASE)).rejects.toThrow(/session source/); + }); + + test("rejects --lookback-days combined with an explicit window", async () => { + await expect( + run([ + ...BASE, + "--lookback-days", + "7", + "--start-time", + "2026-01-01T00:00:00Z", + "--end-time", + "2026-01-02T00:00:00Z", + ]), + ).rejects.toThrow(/cannot be combined/); + }); + + test("rejects a half-open explicit window", async () => { + await expect(run([...BASE, "--start-time", "2026-01-01T00:00:00Z"])).rejects.toThrow( + /together/, + ); + }); + + test("rejects start-time not before end-time", async () => { + await expect( + run([...BASE, "--start-time", "2026-01-02T00:00:00Z", "--end-time", "2026-01-01T00:00:00Z"]), + ).rejects.toThrow(/before/); + }); +}); + +describe("eval ondemand evaluate orchestration", () => { + test("session-ids arm: fetches traces then evaluates them, rendering the result", async () => { + const { core, stdout } = await run([...BASE, "--session-ids", "s1", "s2"]); + + expect(JSON.parse(stdout)).toEqual(RESULT); + + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch?.args[0]).toMatchObject({ + agent: "a-1", + sessionIds: ["s1", "s2"], + window: undefined, + }); + + // evaluate receives exactly the traces getTracesForAgent returned. + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ + traces: [TRACE], + evaluatorIds: ["Builtin.Helpfulness"], + }); + // Order: fetch precedes evaluate. + expect(core.eval.calls.map((c) => c.method)).toEqual(["getTracesForAgent", "evaluate"]); + }); + + test("--trace-id alone is a valid source and is passed to the fetch", async () => { + const { core } = await run([...BASE, "--trace-id", "t1"]); + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch?.args[0]).toMatchObject({ traceId: "t1" }); + }); + + test("--lookback-days resolves to a now-N-days window (start before end)", async () => { + const { core } = await run([...BASE, "--lookback-days", "7"]); + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch).toBeDefined(); + const input = fetch!.args[0] as { window?: { startTime: Date; endTime: Date } }; + expect(input.window).toBeDefined(); + const window = input.window!; + expect(+window.startTime).toBeLessThan(+window.endTime); + const spanDays = (+window.endTime - +window.startTime) / (24 * 60 * 60 * 1000); + expect(spanDays).toBeCloseTo(7, 5); + }); + + test("ground-truth is parsed and passed to evaluate verbatim", async () => { + const groundTruth = [ + { context: { spanContext: { sessionId: "s1" } }, expectedResponse: { text: "hi" } }, + ]; + const { core } = await run([ + ...BASE, + "--session-ids", + "s1", + "--ground-truth", + JSON.stringify(groundTruth), + ]); + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ groundTruth }); + }); +}); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 4a12e554d..c7edd4970 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -25,6 +25,8 @@ import type { ListBatchEvaluationsResponse, StartBatchEvaluationResponse, SessionMetadataShape, + EvaluationReferenceInput, + EvaluationResultContent, DataSourceConfig as DataPlaneDataSourceConfig, } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; @@ -186,6 +188,54 @@ export type StartBatchEvaluationInput = { kmsKeyArn?: string; }; +// SpanRecord is one OTel span/log document — the parsed `@message` JSON of a +// CloudWatch Logs Insights result row. Left open (arbitrary JSON) because it is +// handed to the Evaluate API's `sessionSpans` verbatim; the CLI only reads a few +// well-known fields off it (traceId, spanId, attributes) to route evaluators. +export type SpanRecord = Record; + +// SessionTrace is one session's gathered telemetry, grouped client-side. Neutral +// by design — no Evaluate coupling — so getTracesForAgent stays reusable and +// EvalClient.evaluate owns the mapping into the Evaluate request shape. +export type SessionTrace = { + sessionId: string; // read from attributes.session.id + spans: SpanRecord[]; // full OTel span JSON (@message) + traceIds: string[]; // for TRACE-level evaluators (evaluationTarget.traceIds) + toolCallSpanIds: string[]; // for TOOL_CALL-level evaluators (evaluationTarget.spanIds) +}; + +// GetTracesInput selects which sessions' traces to read for one agent. `sessionIds` +// and `traceId` are independent, optional, AND-ed query filters; with neither, the +// `window` bounds discovery. `window` unset ⇒ the client defaults to now−7d. +export type GetTracesInput = { + agent: string; + endpoint?: string; + window?: SessionWindow; + sessionIds?: string[]; + traceId?: string; +}; + +// EvaluateInput is the CLI-facing shape for the synchronous Evaluate path. +// `groundTruth` is the SDK-native array, passed verbatim; EvalClient groups it by +// session (context.spanContext.sessionId) and attaches per session. +export type EvaluateInput = { + traces: SessionTrace[]; + evaluatorIds: string[]; + groundTruth?: EvaluationReferenceInput[]; +}; + +// EvaluateResult returns the raw Evaluate API results across all evaluators and +// sessions (each carries its own evaluatorId + span context). No aggregation — the +// caller renders the raw scores. The two counts are distinct on purpose: +// `sessionsRequested` is how many gathered sessions were handed to Evaluate; +// `sessionsEvaluated` is how many actually produced results (a TRACE/TOOL_CALL +// session with no matching ids is requested but not evaluated). +export type EvaluateResult = { + sessionsRequested: number; + sessionsEvaluated: number; + results: EvaluationResultContent[]; +}; + // CoreEvalClient is the evaluator, online evaluation, and dataset surface the eval // handlers depend on. It is declared here, next to the handlers that consume it, // and implemented by src/core/eval.tsx (dependency inversion: handlers own the @@ -237,6 +287,16 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; + // getTracesForAgent resolves the agent to its runtime log group and reads the + // target sessions' traces client-side (CloudWatch Logs Insights), grouped by + // session. Returns neutral SessionTrace records — the ondemand handler hands + // them to evaluate. Kept off the Evaluate path so the same fetch is reusable. + getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise; + // evaluate runs evaluators synchronously over already-gathered traces via the + // Evaluate API and returns per-session scores. No job, no CloudWatch — the + // trace read happened in getTracesForAgent. + evaluate(input: EvaluateInput, options: CoreOptions): Promise; + createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index df3414166..0f10e3790 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -110,8 +110,12 @@ import type { CoreEvalClient, CreateDatasetInput, CreateOnlineEvalInput, + EvaluateInput, + EvaluateResult, GetBatchEvaluationResult, + GetTracesInput, LlmAsAJudgeUpdate, + SessionTrace, StartBatchEvaluationInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; @@ -1222,6 +1226,12 @@ export class TestEvalClient implements CoreEvalClient { private batchEvalResults: BatchEvaluationResultEntry[] = []; private batchEvalResultsError?: unknown; private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE; + private getTracesResponse: SessionTrace[] = []; + private evaluateResponse: EvaluateResult = { + sessionsRequested: 0, + sessionsEvaluated: 0, + results: [], + }; private error?: Error; // setListResponse sets what listEvaluators resolves to (when not erroring). @@ -1468,6 +1478,31 @@ export class TestEvalClient implements CoreEvalClient { return this.startBatchEvalResponse; } + // setGetTracesResponse sets what getTracesForAgent resolves to (when not + // erroring). + setGetTracesResponse(traces: SessionTrace[]): this { + this.getTracesResponse = traces; + return this; + } + + // setEvaluateResponse sets what evaluate resolves to (when not erroring). + setEvaluateResponse(response: EvaluateResult): this { + this.evaluateResponse = response; + return this; + } + + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { + this.calls.push({ method: "getTracesForAgent", args: [input, options] }); + if (this.error) throw this.error; + return this.getTracesResponse; + } + + async evaluate(input: EvaluateInput, options: CoreOptions): Promise { + this.calls.push({ method: "evaluate", args: [input, options] }); + if (this.error) throw this.error; + return this.evaluateResponse; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions,