diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 55a57d24f..816ed8463 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -52,20 +52,35 @@ import { type DataSourceConfig, type ListOnlineEvaluationConfigsResponse, type Rule, + type EvaluatorLevel, type UpdateConfigurationBundleResponse, type UpdateEvaluatorResponse, type UpdateOnlineEvaluationConfigResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; import { + EvaluateCommand, GetBatchEvaluationCommand, ListBatchEvaluationsCommand, StartBatchEvaluationCommand, + type EvaluationReferenceInput, + type EvaluationResultContent, + type EvaluationTarget, type ListBatchEvaluationsResponse, type StartBatchEvaluationResponse, type DataSourceConfig as DataPlaneDataSourceConfig, type CloudWatchFilterConfig, } from "@aws-sdk/client-bedrock-agentcore"; +import { + GetQueryResultsCommand, + ResourceNotFoundException, + StartQueryCommand, + type CloudWatchLogsClient, + type ResultField, +} from "@aws-sdk/client-cloudwatch-logs"; +import type { DocumentType } from "@smithy/types"; import { randomUUID } from "node:crypto"; +import { unlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { basename, dirname, extname, join } from "node:path"; import { Transform } from "node:stream"; import { setTimeout as sleep } from "node:timers/promises"; @@ -86,14 +101,24 @@ import type { CreateConfigurationBundleInput, CreateDatasetInput, CreateOnlineEvalInput, + EvaluateInput, + EvaluateResult, GetBatchEvaluationResult, + GetTracesInput, LlmAsAJudgeUpdate, SessionSourceValue, + SessionTrace, + SimulateInput, + SimulateResult, + SpanRecord, StartBatchEvaluationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, } from "../handlers/eval/types"; -import { atomicWrite, atomicWriteStream, readTextFile } from "../io"; +import { atomicWrite, atomicWriteStream, readTextFile, renderJsonTemplate } from "../io"; +import { invokeRuntime } from "./invokeRuntime"; +import { loadDatasetFile, runScenarios, toSessionMetadata } from "./eval/simulate"; +import { normalizeRuntimeInvokeRequest } from "../handlers/runtime/invoke/request"; import { isTerminalStatus, readEvaluationResults } from "./batchEvaluationResults"; import { applyExampleIds, diffExamples, indexRemoteById, parseJsonl } from "./datasetDiff"; import type { Addition } from "./datasetDiff"; @@ -122,6 +147,19 @@ const ALLOWED_DATASET_STATUSES: ReadonlySet = new Set([ ]); const RETRYABLE_DATASET_STATUSES: ReadonlySet = new Set(["CREATING", "UPDATING"]); +// The shared, account-level OTel span log group. +const SPANS_LOG_GROUP = "aws/spans"; + +// Default discovery window when no explicit --start/--end or --lookback-days is +// given. Mirrors the batch service's now-7d default. +const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000; + +// A hard Evaluate limit: at most 10 trace/span ids per request. +const EVALUATE_TARGET_BATCH = 10; + +// CloudWatch Logs Insights hard ceiling: a query returns at most 100k rows. +const INSIGHTS_MAX_ROWS = 100_000; + // noopLogger is the default for the optional logger arg so callers that don't // need batch-evaluation result-log diagnostics (e.g. dataset-only tests) can // omit it. Production (src/core/index.tsx) injects a real child logger. @@ -384,6 +422,221 @@ export class EvalClient implements CoreEvalClient { }; } + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { + const qualifier = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; + + const { runtimeId, runtimeName } = await resolveAgentToNameAndId( + input.agent, + this.clients, + options, + ); + const logGroupName = runtimeLogGroup(runtimeId, qualifier); + const serviceName = runtimeServiceName(runtimeName, qualifier); + + // CloudWatch Insights takes epoch seconds. Discovery defaults to now-7d when + // no explicit window is given (matches the batch service's default). + const endMs = input.window ? +input.window.endTime : Date.now(); + const startMs = input.window ? +input.window.startTime : endMs - SEVEN_DAYS_MS; + const startSec = Math.floor(startMs / 1000); + const endSec = Math.floor(endMs / 1000); + + const logs = this.clients.logs(toClientConfig(options)); + const queryString = buildSpanQuery(serviceName, input.sessionIds, input.traceId); + + // Runtime group required (missing = agent has no traces); aws/spans optional now + // https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html#observability-configure-unified-traces + const [runtimeRows, sharedRows] = await Promise.all([ + runInsightsQuery(logs, [logGroupName], queryString, startSec, endSec).catch((error) => { + if (error instanceof ResourceNotFoundException) { + throw new InputValidationError( + `No telemetry found for agent "${input.agent}": its runtime log group ${logGroupName} ` + + `does not exist. Ensure the agent has been invoked and emits traces.`, + { meta: { agent: input.agent, logGroupName } }, + ); + } + throw error; + }), + runInsightsQuery(logs, [SPANS_LOG_GROUP], queryString, startSec, endSec).catch((error) => { + if (error instanceof ResourceNotFoundException) return []; + throw error; + }), + ]); + const traces = groupSpansBySession([...sharedRows, ...runtimeRows]); + + // Warn when explicitly requested sessions never showed up in the logs (aged + // out, wrong id, or never emitted) so a caller isn't misled by a partial run. + if (input.sessionIds?.length) { + const found = new Set(traces.map((t) => t.sessionId)); + const missing = input.sessionIds.filter((id) => !found.has(id)); + if (missing.length > 0) { + this.logger.warn(`requested sessions not found in logs: ${missing.join(", ")}`); + } + } + return traces; + } + + async evaluate(input: EvaluateInput, options: CoreOptions): Promise { + const data = this.clients.data(toClientConfig(options)); + const control = this.clients.control(toClientConfig(options)); + const levels = await resolveEvaluatorLevels(input.evaluatorIds, control, options); + const refsBySession = groupRefsBySession(input.groundTruth); + + const results: EvaluationResultContent[] = []; + // Sessions that actually produced Evaluate results — distinct from the sessions + // handed in, since a TRACE/TOOL_CALL session with no matching ids makes no call. + const evaluatedSessions = new Set(); + for (const evaluatorId of input.evaluatorIds) { + const level = levels.get(evaluatorId) ?? "SESSION"; + for (const trace of input.traces) { + // TRACE/TOOL_CALL sessions with no ids at that level contribute no calls + // (empty batch list); SESSION always makes one call with no target. + for (const target of targetBatches(level, trace)) { + const response = await data.send( + new EvaluateCommand({ + evaluatorId, + // SpanRecord is Record for ergonomic reads; the API + // wants DocumentType[] (assignable to it, so a single cast suffices). + evaluationInput: { sessionSpans: trace.spans as DocumentType[] }, + evaluationTarget: target, + evaluationReferenceInputs: refsBySession.get(trace.sessionId), + }), + ); + const evaluationResults = response.evaluationResults ?? []; + if (evaluationResults.length > 0) evaluatedSessions.add(trace.sessionId); + results.push(...evaluationResults); + } + } + } + return { + sessionsRequested: input.traces.length, + sessionsEvaluated: evaluatedSessions.size, + results, + }; + } + + async simulate( + input: SimulateInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + // Load scenarios: a local JSONL path directly, else download the dataset id. + let tempDatasetPath: string | undefined; + const path = (await Bun.file(input.dataset).exists()) + ? input.dataset + : (tempDatasetPath = await this.downloadDatasetToTemp( + input.dataset, + input.datasetVersion, + options, + signal, + )); + try { + const scenarios = await loadDatasetFile(path); + const byId = new Map(scenarios.map((s) => [s.scenarioId, s])); + + // Resolve the runtime once; normalizeRuntimeInvokeRequest validates auth + fills + // accountId. Invoke is the extracted free fn — no RuntimeClient dependency. + const runtime = await this.clients + .control(toClientConfig(options)) + .send(new GetAgentRuntimeCommand({ agentRuntimeId: input.runtimeId })); + const deps = { clients: this.clients, fetch: this.fetch, logger: this.logger }; + + const { ok, failed, firstError } = await runScenarios(scenarios, async (scenario) => { + // One session per scenario; the id is generated client-side (session id is a + // client-owned input per the AgentCore docs, needed on the request before any + // response and reused across turns). Turns run sequentially against the SAME + // session so the conversation — and its per-turn traces — accumulate in order, + // matching the per-turn ground truth in toSessionMetadata. + const runtimeSessionId = randomUUID(); + try { + for (const turn of scenario.turns) { + const request = normalizeRuntimeInvokeRequest(runtime, { + runtimeId: input.runtimeId, + qualifier: input.qualifier, + payload: renderJsonTemplate(input.payloadTemplate, { input: turn.input }), + contentType: "application/json", + accept: "application/json", + applicationHeaders: input.headers, + bearerToken: input.bearerToken, + runtimeSessionId, + runtimeUserId: input.userId, + }); + const response = await invokeRuntime(deps, request, options, signal); + for await (const _chunk of response.body) { + // Drain each turn's stream so it completes before the next turn. + } + } + } catch (error) { + this.logger.debug( + `simulate: invoke failed for scenario "${scenario.scenarioId}": ${(error as Error).message}`, + ); + throw error; + } + return { scenarioId: scenario.scenarioId, sessionId: runtimeSessionId }; + }); + + if (ok.length === 0) { + const detail = firstError ? `; first error: ${firstError.message}` : ""; + throw new InputValidationError( + `no scenarios could be invoked (${failed} failed) — nothing to evaluate${detail}`, + ); + } + if (failed > 0) { + this.logger.warn(`simulate: ${failed} scenario(s) failed to invoke and were dropped`); + } + + // AgentCore takes ~30s-3min to emit spans for a freshly invoked session; submit + // too early and the service reads an empty log group and marks every session + // failed. Wait once for the batch to be safely ingestible (matches the old CLI's + // 180s wait). Skipped when disabled via `SIMULATE_INGESTION_WAIT_MS=0` (tests). + const waitMs = Number(process.env.SIMULATE_INGESTION_WAIT_MS ?? 180_000); + if (waitMs > 0) { + this.logger.info( + `waiting ${Math.round(waitMs / 1000)}s for span ingestion before submitting`, + ); + await sleep(waitMs, undefined, { signal }); + } + + const job = await this.startBatchEvaluation( + { + name: input.name, + description: input.description, + evaluatorIds: input.evaluatorIds, + source: { + origin: "agent", + agent: input.runtimeId, + endpoint: input.qualifier, + sessionIds: ok.map((r) => r.sessionId), + }, + groundTruth: ok.map((r) => toSessionMetadata(byId.get(r.scenarioId)!, r.sessionId)), + kmsKeyArn: input.kmsKeyArn, + }, + options, + ); + + return { + batchEvaluationId: job.batchEvaluationId, + status: job.status, + scenariosInvoked: ok.length, + scenariosFailed: failed, + }; + } finally { + if (tempDatasetPath) await unlink(tempDatasetPath).catch(() => {}); + } + } + + // downloadDatasetToTemp streams a dataset version's JSONL to a temp file so + // loadDatasetFile can read it — reuses downloadDataset rather than re-fetching. + private async downloadDatasetToTemp( + id: string, + version: string | undefined, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + const path = join(tmpdir(), `agentcore-dataset-${randomUUID()}.jsonl`); + await this.downloadDataset(id, version, path, options, signal); + return path; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, @@ -1261,6 +1514,221 @@ async function agentDataSource( }; } +// sanitizeQueryValue strips single quotes so an id can't break out of the quoted +// Insights filter literal it is interpolated into (matches the old CLI). +function sanitizeQueryValue(value: string): string { + return value.replace(/'/g, ""); +} + +// buildSpanQuery is the single-phase Insights query: scope to one runtime by its +// OTel service.name, optionally narrow to specific sessions and/or one trace, and +// select the full span JSON (@message) plus the session id to group by. It does +// NOT over-filter on ispresent(kind) — that span-only predicate is what forced the +// old CLI's second query for log records; the looser scope returns everything for +// the session in one pass. +function buildSpanQuery(serviceName: string, sessionIds?: string[], traceId?: string): string { + let query = `fields @message, attributes.session.id as sessionId, traceId, spanId + | filter resource.attributes.service.name in ['${sanitizeQueryValue(serviceName)}']`; + if (sessionIds && sessionIds.length > 0) { + const ids = sessionIds.map((id) => `'${sanitizeQueryValue(id)}'`).join(", "); + query += `\n | filter attributes.session.id in [${ids}]`; + } + if (traceId) { + query += `\n | filter traceId = '${sanitizeQueryValue(traceId)}'`; + } + query += `\n | sort @timestamp asc\n | limit ${INSIGHTS_MAX_ROWS}`; + return query; +} + +// runInsightsQuery starts a CloudWatch Logs Insights query, waits for it to finish, +// then drains all result pages. GetQueryResults returns <=10k rows per call, so a +// long session's spans span multiple pages (nextToken); dropping any would score a +// partial conversation. Fails fast if the ceiling is hit — that belongs in batch. +async function runInsightsQuery( + logs: CloudWatchLogsClient, + logGroupNames: string[], + queryString: string, + startSec: number, + endSec: number, +): Promise { + const started = await logs.send( + new StartQueryCommand({ logGroupNames, queryString, startTime: startSec, endTime: endSec }), + ); + const queryId = started.queryId; + + // Phase 1: wait for completion. A large scan can take minutes, so the deadline is + // generous; each poll costs one cheap GetQueryResults call. + let status = "Running"; + for (let i = 0; i < 300 && status !== "Complete"; i++) { + const result = await logs.send(new GetQueryResultsCommand({ queryId })); + status = result.status ?? "Unknown"; + if (status === "Failed" || status === "Cancelled" || status === "Timeout") { + throw new NetworkingError(`CloudWatch Logs Insights query ${status.toLowerCase()}`, { + meta: { queryId }, + }); + } + if (status !== "Complete") await new Promise((resolve) => setTimeout(resolve, 1000)); + } + if (status !== "Complete") { + throw new NetworkingError("CloudWatch Logs Insights query did not finish in time", { + meta: { queryId }, + }); + } + + // Phase 2: drain pages. Terminates on nextToken; total is bounded by the query's + // `| limit INSIGHTS_MAX_ROWS`. + const rows: ResultField[][] = []; + let nextToken: string | undefined; + do { + const result = await logs.send(new GetQueryResultsCommand({ queryId, nextToken })); + rows.push(...(result.results ?? [])); + nextToken = result.nextToken; + } while (nextToken); + + if (rows.length >= INSIGHTS_MAX_ROWS) { + throw new InputValidationError( + `Too many spans in scope (>= ${INSIGHTS_MAX_ROWS}). Narrow --session-ids or the time ` + + `window, or use 'eval batch-evaluation' for large jobs.`, + ); + } + return rows; +} + +// Group parsed @message docs by session, keeping only sessions with >=1 span +// (Evaluate rejects log-only sessions), and derive each session's trace/tool ids. +function groupSpansBySession(rows: ResultField[][]): SessionTrace[] { + const docsBySession = new Map(); + const sessionsWithSpans = new Set(); + for (const row of rows) { + const message = row.find((f) => f.field === "@message")?.value; + const sessionId = row.find((f) => f.field === "sessionId")?.value; + // Drop orphan records with no session id (e.g. system logs) — an + // unidentifiable session can't be evaluated. + if (!message || !sessionId) continue; + let doc: SpanRecord; + try { + doc = JSON.parse(message) as SpanRecord; + } catch { + continue; + } + const list = docsBySession.get(sessionId); + if (list) list.push(doc); + else docsBySession.set(sessionId, [doc]); + // distinquish between log record and span (which has "kind") + if ("kind" in doc) sessionsWithSpans.add(sessionId); + } + return [...docsBySession] + .filter(([sessionId]) => sessionsWithSpans.has(sessionId)) + .map(([sessionId, spans]) => ({ + sessionId, + spans, + traceIds: extractTraceIds(spans), + toolCallSpanIds: extractToolCallSpanIds(spans), + })); +} + +// extractTraceIds pulls the distinct trace ids out of a session's spans, preserving +// first-seen order (ported from the old CLI's span-collector). +function extractTraceIds(spans: SpanRecord[]): string[] { + const seen = new Set(); + const traceIds: string[] = []; + for (const span of spans) { + const traceId = span.traceId; + // Skip empty ids: log records not tied to a trace carry traceId "", and the + // Evaluate API rejects any target id that isn't a 32-char trace id. + if (typeof traceId === "string" && traceId.length > 0 && !seen.has(traceId)) { + seen.add(traceId); + traceIds.push(traceId); + } + } + return traceIds; +} + +// isToolSpan classifies a tool-execution span by the framework markers the AgentCore +// evaluation SDK uses (Strands / OpenInference / Traceloop). Exact, case-sensitive — +// OpenInference is "TOOL" (upper), Traceloop is "tool" (lower). +function isToolSpan(attrs: Record): boolean { + return ( + attrs["gen_ai.operation.name"] === "execute_tool" || + attrs["openinference.span.kind"] === "TOOL" || + attrs["traceloop.span.kind"] === "tool" + ); +} + +// extractToolCallSpanIds pulls the span ids of tool-execution spans for TOOL_CALL +// evaluators. A tool name attribute alone is unreliable (Traceloop names tools via +// traceloop.entity.name, not tool.name), so classify by span kind instead. +function extractToolCallSpanIds(spans: SpanRecord[]): string[] { + const spanIds: string[] = []; + for (const span of spans) { + const spanId = span.spanId; + if (typeof spanId !== "string" || spanId.length === 0) continue; + if (isToolSpan((span.attributes ?? {}) as Record)) spanIds.push(spanId); + } + return spanIds; +} + +// resolveEvaluatorLevels maps each evaluator id to its evaluation level via +// GetEvaluator — authoritative for both builtins (e.g. Builtin.Helpfulness ⇒ TRACE) +// and custom evaluators, so the level (which decides whether the Evaluate call +// targets trace ids, tool-call span ids, or the whole session) is never guessed. +// GetEvaluator errors propagate: a failed lookup (e.g. AccessDenied, not-found) +// must surface, not silently degrade to SESSION and submit the wrong scope. +async function resolveEvaluatorLevels( + evaluatorIds: string[], + control: BedrockAgentCoreControlClient, + _options: CoreOptions, +): Promise> { + const levels = new Map(); + for (const id of new Set(evaluatorIds)) { + const evaluator = await control.send(new GetEvaluatorCommand({ evaluatorId: id })); + // level is a required response field (SDK types it `| undefined` without `?`). + levels.set(id, evaluator.level!); + } + return levels; +} + +// groupRefsBySession indexes ground-truth reference inputs by the session they +// apply to (context.spanContext.sessionId), so each Evaluate call attaches only its +// own session's references — the synchronous Evaluate shape, not batch's job-level +// evaluationMetadata. +function groupRefsBySession( + groundTruth: EvaluationReferenceInput[] | undefined, +): Map { + const map = new Map(); + for (const ref of groundTruth ?? []) { + const sessionId = + ref.context && "spanContext" in ref.context ? ref.context.spanContext?.sessionId : undefined; + if (!sessionId) continue; + const list = map.get(sessionId); + if (list) list.push(ref); + else map.set(sessionId, [ref]); + } + return map; +} + +// targetBatches splits a session's evaluation targets into <=10-id Evaluate calls. +// SESSION evaluators make a single call with no target; TRACE/TOOL_CALL sessions +// with no ids at that level make none (the session is skipped for that evaluator). +function targetBatches( + level: EvaluatorLevel, + trace: SessionTrace, +): (EvaluationTarget | undefined)[] { + if (level === "TRACE") { + return chunk(trace.traceIds, EVALUATE_TARGET_BATCH).map((traceIds) => ({ traceIds })); + } + if (level === "TOOL_CALL") { + return chunk(trace.toolCallSpanIds, EVALUATE_TARGET_BATCH).map((spanIds) => ({ spanIds })); + } + return [undefined]; +} + +function chunk(items: T[], size: number): T[][] { + const out: T[][] = []; + for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size)); + return out; +} + // A just-written role or inline policy is not visible to the service immediately // (IAM is eventually consistent), and the service validates both when the config // is created. It surfaces as one of two messages depending on which part has not diff --git a/src/core/eval/simulate.ts b/src/core/eval/simulate.ts new file mode 100644 index 000000000..81fc3f843 --- /dev/null +++ b/src/core/eval/simulate.ts @@ -0,0 +1,120 @@ +import type { SessionMetadataShape } from "@aws-sdk/client-bedrock-agentcore"; +import { InputValidationError } from "../../errors"; + +// Scenario is one dataset row for replay. Local to this module on purpose — it is +// not a handler contract, so it does not live in handlers/eval/types.tsx. Field +// names mirror the dataset JSONL (snake_case in, camelCase here). +export type Scenario = { + scenarioId: string; + turns: { input: string; expectedResponse?: string }[]; + assertions?: string[]; + expectedTrajectory?: string[]; +}; + +// parseScenarios reads dataset JSONL (one scenario per line) into Scenario records. +// Each scenario needs a non-empty, unique `scenario_id` — the id is the join key +// between the session created for it and its ground truth, so a missing/duplicate +// id silently misassigns ground truth to the wrong session. +export function parseScenarios(text: string): Scenario[] { + const scenarios: Scenario[] = []; + const seen = new Set(); + for (const line of text.split("\n")) { + const trimmed = line.trim(); + if (!trimmed) continue; + let row: Record; + try { + row = JSON.parse(trimmed) as Record; + } catch { + throw new InputValidationError("dataset contains a line that is not valid JSON"); + } + const scenario = toScenario(row); + if (!scenario.scenarioId) { + throw new InputValidationError("dataset scenario is missing 'scenario_id'"); + } + if (seen.has(scenario.scenarioId)) { + throw new InputValidationError( + `dataset has a duplicate scenario_id: "${scenario.scenarioId}"`, + ); + } + if (scenario.turns.length === 0) { + throw new InputValidationError(`scenario "${scenario.scenarioId}" has no turns`); + } + seen.add(scenario.scenarioId); + scenarios.push(scenario); + } + if (scenarios.length === 0) throw new InputValidationError("dataset has no scenarios"); + return scenarios; +} + +function toScenario(row: Record): Scenario { + const turns = Array.isArray(row.turns) ? row.turns : []; + return { + scenarioId: String(row.scenario_id ?? ""), + turns: turns.map((t: Record) => ({ + input: String(t.input ?? ""), + expectedResponse: t.expected_response as string | undefined, + })), + assertions: row.assertions as string[] | undefined, + expectedTrajectory: row.expected_trajectory as string[] | undefined, + }; +} + +// loadDatasetFile reads scenarios from a local JSONL path. The dataset-id path is +// handled by the caller (downloadDataset to a temp file, then this). +export async function loadDatasetFile(path: string): Promise { + return parseScenarios(await Bun.file(path).text()); +} + +// runScenarios runs `worker` over every scenario with bounded concurrency. A failed +// worker doesn't sink the run — the failure is captured (so the caller can report +// on all-failed) but drops that scenario. Returns ok results + the first error we +// saw, which the caller can surface to explain a total failure. +export type ScenarioRun = { ok: T[]; failed: number; firstError?: Error }; +export async function runScenarios( + scenarios: Scenario[], + worker: (scenario: Scenario) => Promise, + concurrency = 5, +): Promise> { + const ok: T[] = []; + let failed = 0; + let firstError: Error | undefined; + let next = 0; + const run = async (): Promise => { + while (next < scenarios.length) { + const scenario = scenarios[next++]!; + try { + ok.push(await worker(scenario)); + } catch (error) { + failed++; + if (!firstError) firstError = error instanceof Error ? error : new Error(String(error)); + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, scenarios.length) }, run)); + return { ok, failed, firstError }; +} + +// toSessionMetadata maps a scenario's ground truth onto the session the replay +// created — the batch service's per-session ground-truth shape (inline arm). +// Omit array fields when empty: the service rejects zero-length `turns` / +// `assertions` (`length >= 1`) rather than treating an empty array as "no data". +export function toSessionMetadata(scenario: Scenario, sessionId: string): SessionMetadataShape { + const turns = scenario.turns + .filter((t) => t.expectedResponse !== undefined) + .map((t) => ({ expectedResponse: { text: t.expectedResponse! } })); + const assertions = scenario.assertions?.map((text) => ({ text })); + return { + sessionId, + testScenarioId: scenario.scenarioId, + groundTruth: { + inline: { + ...(assertions && assertions.length > 0 && { assertions }), + ...(scenario.expectedTrajectory && + scenario.expectedTrajectory.length > 0 && { + expectedTrajectory: { toolNames: scenario.expectedTrajectory }, + }), + ...(turns.length > 0 && { turns }), + }, + }, + }; +} diff --git a/src/core/invokeRuntime.ts b/src/core/invokeRuntime.ts new file mode 100644 index 000000000..4f9f92e4a --- /dev/null +++ b/src/core/invokeRuntime.ts @@ -0,0 +1,177 @@ +import { randomUUID } from "node:crypto"; +import { InvokeAgentRuntimeCommand } from "@aws-sdk/client-bedrock-agentcore"; +import type { RuntimeInvokeRequest, RuntimeInvokeResponse } from "../handlers/runtime/types"; +import type { Logger } from "../logging"; +import type { AwsClients, CoreFetch, CoreOptions } from "./types"; +import { abortable } from "./abortable"; +import { toClientConfig } from "./utils"; + +// InvokeRuntimeDeps is the slice of a Core client an invoke needs. Passed in as a +// bag (not a sibling client) so both RuntimeClient and EvalClient.simulate can call +// these free functions off their own `this.clients`/`this.fetch`/`this.logger`. +export type InvokeRuntimeDeps = { + clients: AwsClients; + fetch: CoreFetch; + logger: Logger; +}; + +async function* emptyBody(): AsyncGenerator {} + +// invokeRuntime dispatches by auth mode: a bearer token routes to the CUSTOM_JWT +// (raw fetch) path, otherwise the SigV4 SDK path. +export async function invokeRuntime( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + options: CoreOptions, + signal?: AbortSignal, +): Promise { + const { runtimeId, bearerToken } = request; + if (bearerToken !== undefined) { + const logger = deps.logger.child({ + operation: "invokeRuntime", + authMode: "CUSTOM_JWT", + runtimeId, + qualifier: request.qualifier, + region: options.region, + }); + return invokeRuntimeWithCustomJwt(deps, request, bearerToken, options, logger, signal); + } + return invokeRuntimeWithIam(deps, request, options, signal); +} + +async function invokeRuntimeWithCustomJwt( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + bearerToken: string, + options: CoreOptions, + logger: Logger, + signal?: AbortSignal, +): Promise { + const client = deps.clients.data(toClientConfig(options)); + const endpoint = client.config.endpointProvider({ + Region: options.region, + Endpoint: options.endpointUrl, + }); + const url = new URL(endpoint.url); + if (url.protocol !== "https:") { + throw new TypeError("CUSTOM_JWT requires an HTTPS endpoint"); + } + url.pathname = `${url.pathname.replace(/\/?$/, "/")}runtimes/${encodeURIComponent(request.runtimeId)}/invocations`; + url.search = new URLSearchParams({ + accountId: request.accountId, + qualifier: request.qualifier, + }).toString(); + const headers = new Headers(request.applicationHeaders); + try { + headers.set("Authorization", `Bearer ${bearerToken}`); + } catch { + throw new TypeError("Invalid bearer token"); + } + try { + for (const [name, value] of [ + ["Content-Type", request.contentType], + ["Accept", request.accept], + ["Mcp-Session-Id", request.mcpSessionId], + ["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id", request.runtimeSessionId ?? randomUUID()], + ["Mcp-Protocol-Version", request.mcpProtocolVersion], + ["Mcp-Method", request.mcpMethod], + ["Mcp-Name", request.mcpName], + ["X-Amzn-Bedrock-AgentCore-Runtime-User-Id", request.runtimeUserId], + ["X-Amzn-Trace-Id", request.traceId], + ["traceparent", request.traceParent], + ["tracestate", request.traceState], + ["baggage", request.baggage], + ] as const) { + if (value !== undefined) headers.set(name, value); + } + } catch { + throw new TypeError("Invalid Runtime request header"); + } + let response: Response; + try { + response = await deps.fetch(url, { + method: "POST", + redirect: "error", + headers, + body: request.payload as RequestInit["body"], + signal, + }); + } catch (error) { + if (signal?.aborted) throw signal.reason ?? error; + logger + .child({ + errorName: + error instanceof TypeError + ? "TypeError" + : error instanceof Error + ? "Error" + : typeof error, + }) + .debug("Runtime invocation transport failed"); + throw new Error("Runtime invocation failed"); + } + if (!response.ok) { + logger + .child({ httpStatusCode: response.status }) + .debug("Runtime invocation returned a non-success response"); + await response.body?.cancel().catch(() => undefined); + throw new Error(`HTTP ${response.status}`); + } + const body = (response.body as AsyncIterable | null) ?? emptyBody(); + return { + statusCode: response.status, + contentType: response.headers.get("content-type") ?? "", + runtimeSessionId: + response.headers.get("x-amzn-bedrock-agentcore-runtime-session-id") ?? undefined, + mcpSessionId: response.headers.get("mcp-session-id") ?? undefined, + mcpProtocolVersion: response.headers.get("mcp-protocol-version") ?? undefined, + traceId: response.headers.get("x-amzn-trace-id") ?? undefined, + traceParent: response.headers.get("traceparent") ?? undefined, + traceState: response.headers.get("tracestate") ?? undefined, + baggage: response.headers.get("baggage") ?? undefined, + body: signal ? abortable(body, signal) : body, + }; +} + +async function invokeRuntimeWithIam( + deps: InvokeRuntimeDeps, + request: RuntimeInvokeRequest, + options: CoreOptions, + signal?: AbortSignal, +): Promise { + const { runtimeId, applicationHeaders, bearerToken: _bearerToken, ...input } = request; + const command = new InvokeAgentRuntimeCommand({ ...input, agentRuntimeArn: runtimeId }); + if (applicationHeaders?.length) { + command.middlewareStack.add( + (next) => async (args) => { + const sdkRequest = args.request as { headers: Record }; + for (const [name, value] of applicationHeaders) sdkRequest.headers[name] = value; + return next(args); + }, + { step: "build", name: "runtimeApplicationHeaders" }, + ); + } + let response; + try { + response = await deps.clients.data(toClientConfig(options)).send(command, { + abortSignal: signal, + }); + } catch (error) { + if (signal?.aborted) throw signal.reason ?? error; + throw error; + } + + const body = (response.response as AsyncIterable | undefined) ?? emptyBody(); + return { + statusCode: response.statusCode ?? 0, + contentType: response.contentType ?? "", + runtimeSessionId: response.runtimeSessionId, + mcpSessionId: response.mcpSessionId, + mcpProtocolVersion: response.mcpProtocolVersion, + traceId: response.traceId, + traceParent: response.traceParent, + traceState: response.traceState, + baggage: response.baggage, + body: signal ? abortable(body, signal) : body, + }; +} diff --git a/src/core/runtime.tsx b/src/core/runtime.tsx index 5de368453..1a3ed2027 100644 --- a/src/core/runtime.tsx +++ b/src/core/runtime.tsx @@ -1,4 +1,3 @@ -import { randomUUID } from "node:crypto"; import { GetAgentRuntimeCommand, GetAgentRuntimeEndpointCommand, @@ -11,7 +10,6 @@ import { type ListAgentRuntimesResponse, type ListAgentRuntimeVersionsResponse, } from "@aws-sdk/client-bedrock-agentcore-control"; -import { InvokeAgentRuntimeCommand } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreRuntimeClient, RuntimeInvokeRequest, @@ -19,11 +17,9 @@ import type { } from "../handlers/runtime/types"; import type { Logger } from "../logging"; import type { AwsClients, CoreFetch, CoreOptions } from "./types"; -import { abortable } from "./abortable"; +import { invokeRuntime } from "./invokeRuntime"; import { toClientConfig } from "./utils"; -async function* emptyBody(): AsyncGenerator {} - export class RuntimeClient implements CoreRuntimeClient { constructor( private readonly clients: AwsClients, @@ -31,158 +27,20 @@ export class RuntimeClient implements CoreRuntimeClient { private readonly logger: Logger, ) {} - async invokeRuntime( - request: RuntimeInvokeRequest, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const { runtimeId, bearerToken } = request; - if (bearerToken !== undefined) { - const logger = this.logger.child({ - operation: "invokeRuntime", - authMode: "CUSTOM_JWT", - runtimeId, - qualifier: request.qualifier, - region: options.region, - }); - return this.invokeRuntimeWithCustomJwt(request, bearerToken, options, logger, signal); - } - return this.invokeRuntimeWithIam(request, options, signal); - } - - private async invokeRuntimeWithCustomJwt( + // invokeRuntime delegates to the free function so EvalClient.simulate can reuse + // the same invoke logic without holding a RuntimeClient (both call it off their + // own clients/fetch/logger). + invokeRuntime( request: RuntimeInvokeRequest, - bearerToken: string, options: CoreOptions, - logger: Logger, signal?: AbortSignal, ): Promise { - const client = this.clients.data(toClientConfig(options)); - const endpoint = client.config.endpointProvider({ - Region: options.region, - Endpoint: options.endpointUrl, - }); - const url = new URL(endpoint.url); - if (url.protocol !== "https:") { - throw new TypeError("CUSTOM_JWT requires an HTTPS endpoint"); - } - url.pathname = `${url.pathname.replace(/\/?$/, "/")}runtimes/${encodeURIComponent(request.runtimeId)}/invocations`; - url.search = new URLSearchParams({ - accountId: request.accountId, - qualifier: request.qualifier, - }).toString(); - const headers = new Headers(request.applicationHeaders); - try { - headers.set("Authorization", `Bearer ${bearerToken}`); - } catch { - throw new TypeError("Invalid bearer token"); - } - try { - for (const [name, value] of [ - ["Content-Type", request.contentType], - ["Accept", request.accept], - ["Mcp-Session-Id", request.mcpSessionId], - ["X-Amzn-Bedrock-AgentCore-Runtime-Session-Id", request.runtimeSessionId ?? randomUUID()], - ["Mcp-Protocol-Version", request.mcpProtocolVersion], - ["Mcp-Method", request.mcpMethod], - ["Mcp-Name", request.mcpName], - ["X-Amzn-Bedrock-AgentCore-Runtime-User-Id", request.runtimeUserId], - ["X-Amzn-Trace-Id", request.traceId], - ["traceparent", request.traceParent], - ["tracestate", request.traceState], - ["baggage", request.baggage], - ] as const) { - if (value !== undefined) headers.set(name, value); - } - } catch { - throw new TypeError("Invalid Runtime request header"); - } - let response: Response; - try { - response = await this.fetch(url, { - method: "POST", - redirect: "error", - headers, - body: request.payload as RequestInit["body"], - signal, - }); - } catch (error) { - if (signal?.aborted) throw signal.reason ?? error; - logger - .child({ - errorName: - error instanceof TypeError - ? "TypeError" - : error instanceof Error - ? "Error" - : typeof error, - }) - .debug("Runtime invocation transport failed"); - throw new Error("Runtime invocation failed"); - } - if (!response.ok) { - logger - .child({ httpStatusCode: response.status }) - .debug("Runtime invocation returned a non-success response"); - await response.body?.cancel().catch(() => undefined); - throw new Error(`HTTP ${response.status}`); - } - const body = (response.body as AsyncIterable | null) ?? emptyBody(); - return { - statusCode: response.status, - contentType: response.headers.get("content-type") ?? "", - runtimeSessionId: - response.headers.get("x-amzn-bedrock-agentcore-runtime-session-id") ?? undefined, - mcpSessionId: response.headers.get("mcp-session-id") ?? undefined, - mcpProtocolVersion: response.headers.get("mcp-protocol-version") ?? undefined, - traceId: response.headers.get("x-amzn-trace-id") ?? undefined, - traceParent: response.headers.get("traceparent") ?? undefined, - traceState: response.headers.get("tracestate") ?? undefined, - baggage: response.headers.get("baggage") ?? undefined, - body: signal ? abortable(body, signal) : body, - }; - } - - private async invokeRuntimeWithIam( - request: RuntimeInvokeRequest, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const { runtimeId, applicationHeaders, bearerToken: _bearerToken, ...input } = request; - const command = new InvokeAgentRuntimeCommand({ ...input, agentRuntimeArn: runtimeId }); - if (applicationHeaders?.length) { - command.middlewareStack.add( - (next) => async (args) => { - const sdkRequest = args.request as { headers: Record }; - for (const [name, value] of applicationHeaders) sdkRequest.headers[name] = value; - return next(args); - }, - { step: "build", name: "runtimeApplicationHeaders" }, - ); - } - let response; - try { - response = await this.clients.data(toClientConfig(options)).send(command, { - abortSignal: signal, - }); - } catch (error) { - if (signal?.aborted) throw signal.reason ?? error; - throw error; - } - - const body = (response.response as AsyncIterable | undefined) ?? emptyBody(); - return { - statusCode: response.statusCode ?? 0, - contentType: response.contentType ?? "", - runtimeSessionId: response.runtimeSessionId, - mcpSessionId: response.mcpSessionId, - mcpProtocolVersion: response.mcpProtocolVersion, - traceId: response.traceId, - traceParent: response.traceParent, - traceState: response.traceState, - baggage: response.baggage, - body: signal ? abortable(body, signal) : body, - }; + return invokeRuntime( + { clients: this.clients, fetch: this.fetch, logger: this.logger }, + request, + options, + signal, + ); } async getRuntime( diff --git a/src/core/types.tsx b/src/core/types.tsx index 98a2f338d..9b1a2d4a8 100644 --- a/src/core/types.tsx +++ b/src/core/types.tsx @@ -37,7 +37,7 @@ export type CoreFetch = ( // full ClientConfig so callers can request any client customization (region, // endpoint, ...). export interface AwsClients { - control(config: ClientConfig): BedrockAgentCoreControlClient; + control(config: ClientConfig): BedrockAgentCoreControlClient data(config: ClientConfig): BedrockAgentCoreClient; iam(config: ClientConfig): IAMClient; // logs reads the CloudWatch Logs streams AgentCore writes batch-evaluation diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index 9eb348b21..b710f529c 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -60,7 +60,7 @@ describe("eval batch-evaluation command hierarchy", () => { .find((c) => c.name() === "eval") ?.children() .find((c) => c.name() === "batch-evaluation"); - expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "get", "list"]); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate", "simulate", "get", "list"]); }); test("prints help for `eval batch-evaluation --json` without an SDK call", async () => { diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx index da96049ad..3d865cb7c 100644 --- a/src/handlers/eval/batch-evaluation/index.tsx +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -6,6 +6,7 @@ import type { Core } from "../../types"; import { createGetBatchEvaluationHandler } from "./get"; import { createListBatchEvaluationsHandler } from "./list"; import { createEvaluateBatchEvaluationHandler } from "./evaluate"; +import { createSimulateBatchEvaluationHandler } from "./simulate"; // batch-evaluation supports evaluate (start an async job) plus get + list. A bare // invocation opens the interactive TUI (list → get), matching evaluator and @@ -15,6 +16,7 @@ export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { .use(withTuiOnEmptyFlagsAndArgs(core, io)) .default(renderTui(core, io)) .handler(createEvaluateBatchEvaluationHandler(core, io)) + .handler(createSimulateBatchEvaluationHandler(core, io)) .handler(createGetBatchEvaluationHandler(core, io)) .handler(createListBatchEvaluationsHandler(core)); } diff --git a/src/handlers/eval/batch-evaluation/simulate/index.tsx b/src/handlers/eval/batch-evaluation/simulate/index.tsx new file mode 100644 index 000000000..276e8f1b8 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/simulate/index.tsx @@ -0,0 +1,87 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; +import { parseRuntimeInvokeHeaders } from "../../../runtime/invoke/request"; + +// batch-evaluation simulate replays a dataset against a runtime (invoke per scenario) +// then submits a batch evaluation over the sessions it created. Invoke flags mirror +// `runtime invoke`; content-type/accept are fixed to application/json. +export const createSimulateBatchEvaluationHandler = (core: Core, _io: AppIO) => + createHandler({ + name: "simulate", + description: "replay a dataset against a runtime, then batch-evaluate the resulting sessions", + flags: [ + flag("runtime-id", "runtime id to invoke per scenario", z.string().optional()), + flag("qualifier", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag( + "payload-template", + 'JSON payload template; {input} is the scenario input, e.g. {"prompt":"{input}"}', + z.string().optional(), + ), + flag("header", "an ordered application header (repeatable)", z.array(z.string()).optional()), + flag( + "bearer-token", + "CUSTOM_JWT bearer token (for JWT-auth runtimes)", + z.string().optional(), + ), + flag("user-id", "runtime user id", z.string().optional()), + flag("dataset", "dataset source: local JSONL path or a dataset id", z.string().optional()), + flag("dataset-version", "dataset version (with a dataset id)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag("name", "batch evaluation name (unique in the account)", z.string().optional()), + flag("description", "optional description", z.string().optional()), + flag("kms-key-arn", "KMS key to encrypt evaluation data at rest", z.string().optional()), + ], + handle: async (ctx, flags) => { + if (!flags["runtime-id"]) + throw new InputValidationError("required option '--runtime-id' not specified"); + if (!flags["payload-template"]) { + throw new InputValidationError("required option '--payload-template' not specified"); + } + if (!flags["dataset"]) + throw new InputValidationError("required option '--dataset' not specified"); + if (!flags["evaluator"]?.length) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + if (!flags["name"]) + throw new InputValidationError("required option '--name ' not specified"); + + // Ctrl-C aborts the run (invokes, the ingestion wait, the dataset download). + const controller = new AbortController(); + const interrupt = () => controller.abort(); + process.once("SIGINT", interrupt); + try { + const result = await core.eval.simulate( + { + runtimeId: flags["runtime-id"], + qualifier: flags["qualifier"], + payloadTemplate: flags["payload-template"], + headers: parseRuntimeInvokeHeaders(flags["header"]), + bearerToken: flags["bearer-token"], + userId: flags["user-id"], + dataset: flags["dataset"], + datasetVersion: flags["dataset-version"], + evaluatorIds: flags["evaluator"], + name: flags["name"], + description: flags["description"], + kmsKeyArn: flags["kms-key-arn"], + }, + coreOptsFromCtx(ctx), + controller.signal, + ); + ctx.require(JsonRendererKey).renderJson(result); + } catch (error) { + // A Ctrl-C exits quietly; the half-created sessions grade nothing. + if (controller.signal.aborted) return; + throw error; + } finally { + process.off("SIGINT", interrupt); + } + }, + }); diff --git a/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx new file mode 100644 index 000000000..fb0c6149d --- /dev/null +++ b/src/handlers/eval/batch-evaluation/simulate/simulate.test.tsx @@ -0,0 +1,132 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../../index"; +import { + createSilentLogger, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../../testing"; +import type { SimulateResult } from "../../types"; + +const RESULT: SimulateResult = { + batchEvaluationId: "batch-eval-sim", + status: "RUNNING", + scenariosInvoked: 3, + scenariosFailed: 0, +}; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setSimulateResponse(RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout() }; +} + +const BASE = [ + "eval", + "batch-evaluation", + "simulate", + "--runtime-id", + "r-1", + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "Builtin.Helpfulness", + "--name", + "sim-1", +]; + +describe("eval batch-evaluation simulate", () => { + test("registered under batch-evaluation", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "batch-evaluation"); + expect(group?.children().map((c) => c.name())).toContain("simulate"); + }); + + test.each([ + [ + [ + "--payload-template", + '{"prompt":"{input}"}', + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "E", + "--name", + "n", + ], + /--runtime-id/, + ], + [ + ["--runtime-id", "r-1", "--dataset", "/tmp/ds.jsonl", "--evaluator", "E", "--name", "n"], + /--payload-template/, + ], + [ + ["--runtime-id", "r-1", "--payload-template", "{}", "--evaluator", "E", "--name", "n"], + /--dataset/, + ], + [ + [ + "--runtime-id", + "r-1", + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--name", + "n", + ], + /--evaluator/, + ], + [ + [ + "--runtime-id", + "r-1", + "--payload-template", + "{}", + "--dataset", + "/tmp/ds.jsonl", + "--evaluator", + "E", + ], + /--name/, + ], + ])("rejects missing required flag", async (args, expected) => { + await expect(run(["eval", "batch-evaluation", "simulate", ...args])).rejects.toThrow(expected); + }); + + test("maps flags to the simulate input and renders the result", async () => { + const { core, stdout } = await run([...BASE, "--qualifier", "PROD", "--header", "x-a:1"]); + expect(JSON.parse(stdout)).toEqual(RESULT); + const call = core.eval.calls.find((c) => c.method === "simulate"); + expect(call?.args[0]).toMatchObject({ + runtimeId: "r-1", + qualifier: "PROD", + payloadTemplate: '{"prompt":"{input}"}', + headers: [["x-a", "1"]], + dataset: "/tmp/ds.jsonl", + evaluatorIds: ["Builtin.Helpfulness"], + name: "sim-1", + }); + // Handler wires an AbortSignal (Ctrl-C) through to Core. + expect(call?.args[2]).toBeInstanceOf(AbortSignal); + }); +}); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index 5562ce2ce..add16936e 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -7,6 +7,7 @@ import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; import { createDatasetHandler } from "./dataset"; import { createBatchEvaluationHandler } from "./batch-evaluation"; +import { createOnDemandHandler } from "./ondemand"; import { createConfigBundleHandler } from "./config-bundle"; export function createEvalHandler(core: Core, io: AppIO): Router { @@ -17,6 +18,7 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .handler(createOnlineEvalHandler(core, io)) .handler(createDatasetHandler(core, io)) .handler(createBatchEvaluationHandler(core, io)) + .handler(createOnDemandHandler(core, io)) .handler(createConfigBundleHandler(core, io)); } diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json new file mode 100644 index 000000000..31796ea2d --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.76a9b708fd699093.json @@ -0,0 +1,23 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a", + "traceId": "6a7cabfa3bfe9a7348415c7b507648a8" + } + }, + "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 941, + "outputTokens": 118, + "totalTokens": 1059 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json new file mode 100644 index 000000000..f0b13e303 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/EvaluateCommand.c0c383465d2d11bc.json @@ -0,0 +1,23 @@ +{ + "evaluationResults": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", + "traceId": "6a7cac0e3fa42bfe5437eef070d1231c" + } + }, + "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.", + "value": 1, + "label": "Above And Beyond", + "tokenUsage": { + "inputTokens": 859, + "outputTokens": 154, + "totalTokens": 1013 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json new file mode 100644 index 000000000..2d3b5e713 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json @@ -0,0 +1,47 @@ +{ + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeName": "asdf_MyAgent", + "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q", + "agentRuntimeVersion": "1", + "createdAt": { + "$date": "2026-04-23T21:17:21.895Z" + }, + "lastUpdatedAt": { + "$date": "2026-04-23T21:17:35.159Z" + }, + "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK", + "networkConfiguration": { + "networkMode": "PUBLIC" + }, + "status": "READY", + "lifecycleConfiguration": { + "idleRuntimeSessionTimeout": 900, + "maxLifetime": 28800 + }, + "description": "AgentCore Runtime: asdf_MyAgent", + "workloadIdentityDetails": { + "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q" + }, + "agentRuntimeArtifact": { + "codeConfiguration": { + "code": { + "s3": { + "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2", + "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip" + } + }, + "runtime": "PYTHON_3_13", + "entryPoint": [ + "opentelemetry-instrument", + "main.py" + ] + } + }, + "environmentVariables": { + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE", + "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" + }, + "metadataConfiguration": { + "requireMMDSV2": true + } +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json new file mode 100644 index 000000000..1825df228 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json @@ -0,0 +1,51 @@ +{ + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "evaluatorConfig": { + "llmAsAJudge": { + "ratingScale": { + "numerical": [ + { + "value": 0, + "label": "Not helpful at all" + }, + { + "value": 1, + "label": "Very unhelpful" + }, + { + "value": 2, + "label": "Somewhat unhelpful" + }, + { + "value": 3, + "label": "Neutral/Mixed" + }, + { + "value": 4, + "label": "Somewhat helpful" + }, + { + "value": 5, + "label": "Very helpful" + }, + { + "value": 6, + "label": "Above and beyond" + } + ] + } + } + }, + "level": "TRACE", + "status": "ACTIVE", + "createdAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "updatedAt": { + "$date": "2024-10-22T00:00:00.000Z" + }, + "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", + "lockedForModification": true +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json new file mode 100644 index 000000000..f5626155b --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.19164cb5cd9b9b70.json @@ -0,0 +1,586 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"1d3bcfe3653d81e4\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392103874218,\"endTimeUnixNano\":1786555392105289657,\"durationNano\":1415439,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "1d3bcfe3653d81e4" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"11655ee4cdde1872\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392105841503,\"endTimeUnixNano\":1786555392106502618,\"durationNano\":661115,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "11655ee4cdde1872" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"210fa6dea654dcbc\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392106868305,\"endTimeUnixNano\":1786555392107751277,\"durationNano\":882972,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "210fa6dea654dcbc" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"cde6ff0cb3289288\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392241890106,\"endTimeUnixNano\":1786555392285594042,\"durationNano\":43703936,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "cde6ff0cb3289288" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"9d3e945db9599945\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392289575652,\"endTimeUnixNano\":1786555392316954181,\"durationNano\":27378529,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "9d3e945db9599945" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"31efc7027314ac76\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392318607342,\"endTimeUnixNano\":1786555392369734375,\"durationNano\":51127033,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "31efc7027314ac76" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"53a0cd3ac657fb83\",\"parentSpanId\":\"db6cce231beea7f0\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555392389258299,\"endTimeUnixNano\":1786555394009438471,\"durationNano\":1620180172,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"c1b5339e-9b84-45b0-a4ad-cfd818de8fca\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"tool_use\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1210,\"aws.genai.token_count_total\":1280,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH3RTZMMT4\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "53a0cd3ac657fb83" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"db6cce231beea7f0\",\"parentSpanId\":\"7f719fd629b95bca\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392374099914,\"endTimeUnixNano\":1786555394010008258,\"durationNano\":1635908344,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1210,\"gen_ai.usage.output_tokens\":70,\"gen_ai.server.request.duration\":1601,\"gen_ai.usage.total_tokens\":1280,\"gen_ai.usage.completion_tokens\":70,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.374107+00:00\",\"gen_ai.server.time_to_first_token\":1613,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.009969+00:00\",\"gen_ai.usage.input_tokens\":1210,\"aws.genai.token_count_total\":1280,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "db6cce231beea7f0" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"26952cbd9a43fe35\",\"parentSpanId\":\"7f719fd629b95bca\",\"flags\":256,\"name\":\"execute_tool add_numbers\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394010521473,\"endTimeUnixNano\":1786555394011315403,\"durationNano\":793930,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.tool.status\":\"success\",\"gen_ai.tool.call.id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\",\"gen_ai.tool.description\":\"Return the sum of two numbers\",\"gen_ai.tool.json_schema\":\"{\\\"properties\\\": {\\\"a\\\": {\\\"description\\\": \\\"Parameter a\\\", \\\"type\\\": \\\"integer\\\"}, \\\"b\\\": {\\\"description\\\": \\\"Parameter b\\\", \\\"type\\\": \\\"integer\\\"}}, \\\"required\\\": [\\\"a\\\", \\\"b\\\"], \\\"type\\\": \\\"object\\\"}\",\"aws.genai.span_kind\":\"TOOL\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.010538+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"execute_tool\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.011292+00:00\",\"gen_ai.tool.name\":\"add_numbers\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "26952cbd9a43fe35" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7f719fd629b95bca\",\"parentSpanId\":\"a5e83c7e772efdf6\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392373929874,\"endTimeUnixNano\":1786555394011480146,\"durationNano\":1637550272,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:14.011468+00:00\",\"event_loop.cycle_id\":\"23f63c39-82fe-4159-8035-ff8e6208b88f\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.373944+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7f719fd629b95bca" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKZ/8hIABqfKwDAAAAMyIAEo6cD/t/8zMNvP/7f/MzgKQKqKAUjxX1CnOCACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"0abec106a8ff39b7\",\"parentSpanId\":\"7576c2384130fece\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555394012454655,\"endTimeUnixNano\":1786555397051836921,\"durationNano\":3039382266,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"8926bd3b-32aa-42a7-a234-5bafc1cd8537\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1294,\"aws.genai.token_count_total\":1306,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWH3RTZMMT4\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "0abec106a8ff39b7" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAAYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7576c2384130fece\",\"parentSpanId\":\"a1d0f9231c5db33d\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394011912861,\"endTimeUnixNano\":1786555397052514047,\"durationNano\":3040601186,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1294,\"gen_ai.usage.output_tokens\":12,\"gen_ai.server.request.duration\":3034,\"gen_ai.usage.total_tokens\":1306,\"gen_ai.usage.completion_tokens\":12,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.011919+00:00\",\"gen_ai.server.time_to_first_token\":2804,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.052476+00:00\",\"gen_ai.usage.input_tokens\":1294,\"aws.genai.token_count_total\":1306,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7576c2384130fece" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAEYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a1d0f9231c5db33d\",\"parentSpanId\":\"a5e83c7e772efdf6\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555394011746512,\"endTimeUnixNano\":1786555397052873967,\"durationNano\":3041127455,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.052853+00:00\",\"event_loop.cycle_id\":\"beebe4f0-27a6-4aef-a0e3-09cda0c61a19\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:14.011757+00:00\",\"event_loop.parent_cycle_id\":\"23f63c39-82fe-4159-8035-ff8e6208b88f\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a1d0f9231c5db33d" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAIYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"5aa1ead48a6ac0e1\",\"parentSpanId\":\"e1355f7dda87635b\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1786555392052368325,\"endTimeUnixNano\":1786555397053469078,\"durationNano\":5001100753,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":55860,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "5aa1ead48a6ac0e1" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAQYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a5e83c7e772efdf6\",\"parentSpanId\":\"5aa1ead48a6ac0e1\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555392372920661,\"endTimeUnixNano\":1786555397053072269,\"durationNano\":4680151608,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":2504,\"gen_ai.usage.output_tokens\":82,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":2586,\"gen_ai.usage.completion_tokens\":82,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:12.372942+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:17.053049+00:00\",\"gen_ai.usage.input_tokens\":2504,\"aws.genai.token_count_total\":2586,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a5e83c7e772efdf6" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAMYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"42f4da466ab4c999\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411324466077,\"endTimeUnixNano\":1786555411324998705,\"durationNano\":532628,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "42f4da466ab4c999" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAYYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"6d003c695538a610\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"PUT\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411323051352,\"endTimeUnixNano\":1786555411324071465,\"durationNano\":1020113,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/api/token\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"PUT /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"PUT\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "6d003c695538a610" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAUYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.urllib3\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"8a05cdd0a93d3a6a\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"GET\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411325315616,\"endTimeUnixNano\":1786555411325825454,\"durationNano\":509838,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"http://169.254.169.254/latest/meta-data/iam/security-credentials/execution_role\",\"aws.remote.service\":\"169.254.169.254\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"GET /latest\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"GET\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "8a05cdd0a93d3a6a" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAcYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"0745de30c628c8e5\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411431858312,\"endTimeUnixNano\":1786555411485160682,\"durationNano\":53302370,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "0745de30c628c8e5" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAgYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"cb5ae6a1e269e21b\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411487794771,\"endTimeUnixNano\":1786555411518235079,\"durationNano\":30440308,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":202,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":202,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "cb5ae6a1e269e21b" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAkYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.httpx\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"b043a295f3c1c4cf\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"POST\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411519590941,\"endTimeUnixNano\":1786555411598192831,\"durationNano\":78601890,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"telemetry.extended\":\"true\",\"http.url\":\"https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp\",\"aws.remote.service\":\"bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"POST /mcp\",\"http.status_code\":200,\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "b043a295f3c1c4cf" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.botocore.bedrock-runtime\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"b556c691098ede16\",\"parentSpanId\":\"627f6364bb0599fa\",\"flags\":256,\"name\":\"chat global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"kind\":\"CLIENT\",\"startTimeUnixNano\":1786555411614408678,\"endTimeUnixNano\":1786555414476372175,\"durationNano\":2861963497,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"rpc.service\":\"Bedrock Runtime\",\"aws.remote.resource.identifier\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"aws.remote.operation\":\"ConverseStream\",\"gen_ai.provider.name\":\"aws.bedrock\",\"server.address\":\"bedrock-runtime.us-west-2.amazonaws.com\",\"aws.request_id\":\"47288f71-5ae9-4b15-9720-d49cf3e180f2\",\"aws.local.operation\":\"UnmappedOperation\",\"aws.span.kind\":\"CLIENT\",\"aws.auth.region\":\"us-west-2\",\"rpc.method\":\"ConverseStream\",\"gen_ai.response.finish_reasons\":[\"end_turn\"],\"server.port\":443,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"http.response.status_code\":200,\"gen_ai.system\":\"aws.bedrock\",\"telemetry.extended\":\"true\",\"gen_ai.usage.output_tokens\":48,\"aws.genai.span_kind\":\"LLM\",\"rpc.system\":\"aws-api\",\"aws.remote.service\":\"AWS::BedrockRuntime\",\"http.status_code\":200,\"aws.region\":\"us-west-2\",\"aws.remote.resource.type\":\"AWS::Bedrock::Model\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"retry_attempts\":0,\"PlatformType\":\"AWS::BedrockAgentCore\",\"aws.auth.account.access_key\":\"ASIAZ7CHXJWHYLSAHMVB\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "b556c691098ede16" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"627f6364bb0599fa\",\"parentSpanId\":\"e90c8e0cb4712af5\",\"flags\":256,\"name\":\"chat\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411601581872,\"endTimeUnixNano\":1786555414476887232,\"durationNano\":2875305360,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":48,\"gen_ai.server.request.duration\":2842,\"gen_ai.usage.total_tokens\":1251,\"gen_ai.usage.completion_tokens\":48,\"aws.genai.span_kind\":\"LLM\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.601589+00:00\",\"gen_ai.server.time_to_first_token\":1966,\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"chat\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.476851+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "627f6364bb0599fa" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEAwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"e90c8e0cb4712af5\",\"parentSpanId\":\"00821fcc8a234519\",\"flags\":256,\"name\":\"execute_event_loop_cycle\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411601446390,\"endTimeUnixNano\":1786555414477214324,\"durationNano\":2875767934,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.operation.name\":\"execute_event_loop_cycle\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.477197+00:00\",\"event_loop.cycle_id\":\"cbaf6d7e-50f0-436e-8c06-6c5d9e6c5936\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.601458+00:00\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "e90c8e0cb4712af5" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA0YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\",\"version\":\"\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"00821fcc8a234519\",\"parentSpanId\":\"82efc476a79eaee2\",\"flags\":256,\"name\":\"invoke_agent Strands Agents\",\"kind\":\"INTERNAL\",\"startTimeUnixNano\":1786555411600856280,\"endTimeUnixNano\":1786555414477366577,\"durationNano\":2876510297,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"gen_ai.usage.prompt_tokens\":1203,\"gen_ai.usage.output_tokens\":48,\"gen_ai.usage.cache_write_input_tokens\":0,\"gen_ai.agent.name\":\"Strands Agents\",\"gen_ai.usage.total_tokens\":1251,\"gen_ai.usage.completion_tokens\":48,\"aws.genai.span_kind\":\"AGENT\",\"gen_ai.event.start_time\":\"2026-08-12T17:23:31.600874+00:00\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"gen_ai.provider.name\":\"strands-agents\",\"gen_ai.operation.name\":\"invoke_agent\",\"gen_ai.event.end_time\":\"2026-08-12T17:23:34.477345+00:00\",\"gen_ai.usage.input_tokens\":1203,\"aws.genai.token_count_total\":1251,\"gen_ai.request.model\":\"global.anthropic.claude-sonnet-4-5-20250929-v1:0\",\"gen_ai.usage.cache_read_input_tokens\":0,\"gen_ai.agent.tools\":\"[\\\"add_numbers\\\", \\\"x_amz_bedrock_agentcore_search\\\", \\\"mcpTarget___web_fetch_exa\\\", \\\"mcpTarget___web_search_exa\\\"]\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"gen_ai.system\":\"strands-agents\"},\"status\":{\"code\":\"OK\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "00821fcc8a234519" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA4YAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"deployment.environment.name\":\"bedrock-agentcore:default\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"cloud.region\":\"us-west-2\",\"aws.log.stream.names\":\"otel-rt-logs\",\"telemetry.sdk.name\":\"opentelemetry\",\"aws.service.type\":\"gen_ai_agent\",\"telemetry.sdk.language\":\"python\",\"cloud.provider\":\"aws\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"telemetry.sdk.version\":\"1.40.0\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"telemetry.auto.version\":\"0.17.0-aws\"}},\"scope\":{\"name\":\"opentelemetry.instrumentation.starlette\",\"version\":\"0.61b0\"},\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"82efc476a79eaee2\",\"parentSpanId\":\"1cc0af0f799f67f1\",\"flags\":768,\"name\":\"POST /invocations\",\"kind\":\"SERVER\",\"startTimeUnixNano\":1786555411282407024,\"endTimeUnixNano\":1786555414477671431,\"durationNano\":3195264407,\"attributes\":{\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"net.peer.port\":34652,\"telemetry.extended\":\"true\",\"http.target\":\"/invocations\",\"http.flavor\":\"1.1\",\"http.url\":\"http://cell01.us-west-2.prod.arp.kepler-analytics.aws.dev/invocations\",\"net.peer.ip\":\"127.0.0.1\",\"http.host\":\"127.0.0.1:8080\",\"aws.local.environment\":\"bedrock-agentcore:default\",\"http.status_code\":200,\"aws.local.operation\":\"POST /invocations\",\"aws.span.kind\":\"SERVER\",\"http.server_name\":\"cell01.us-west-2.prod.arp.kepler-analytics.aws.dev\",\"net.host.port\":8080,\"http.route\":\"/invocations\",\"PlatformType\":\"AWS::BedrockAgentCore\",\"http.method\":\"POST\",\"http.response.status_code\":200,\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"http.scheme\":\"http\"},\"status\":{\"code\":\"UNSET\"}}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "82efc476a79eaee2" + }, + { + "field": "@ptr", + "value": "CpwBCmAKFjY4NTE5NzcwODY4Nzphd3Mvc3BhbnMQABokZTE0NTI5MzgtZjVjZi00OTc1LWE0N2QtYzk3YzMwZmFlY2ZlIg4IgJiTmv8zEOfXrMP/Mzjv37W+yTNAyO+wktMzSAASNhoYAgammbXDAAAAAKaAAVIABqfKwHAAAAMyIAEou+f/t/8zMM3vgLj/MzgQQMPmAUiIclCpRiACEA8YAQ==" + } + ] + ], + "statistics": { + "recordsMatched": 26, + "recordsScanned": 26, + "estimatedRecordsSkipped": 0, + "bytesScanned": 47213, + "estimatedBytesSkipped": 0, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json new file mode 100644 index 000000000..a4401cbb3 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/GetQueryResultsCommand.275f4a33f37e8fc7.json @@ -0,0 +1,212 @@ +{ + "queryLanguage": "CWLI", + "results": [ + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394010008258,\"observedTimeUnixNano\":1786555395351842794,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"tool_use\",\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"db6cce231beea7f0\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "db6cce231beea7f0" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBoYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394011480146,\"observedTimeUnixNano\":1786555395352095155,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\",\"tool.result\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}},{\"role\":\"assistant\",\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7f719fd629b95bca\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7f719fd629b95bca" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBwYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555394011315403,\"observedTimeUnixNano\":1786555395351997309,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"tool\",\"content\":{\"role\":\"tool\",\"content\":\"{\\\"a\\\": 2, \\\"b\\\": 2}\",\"id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"[{\\\"text\\\": \\\"4\\\"}]\",\"id\":\"tooluse_wZxoTzan8sypzmeYPveFwA\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"26952cbd9a43fe35\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "26952cbd9a43fe35" + }, + { + "field": "@ptr", + "value": "CswBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNhoYAgaNwSWxAAAABgdxOn0ABqfKutAAAAASIAEo47b/t/8zMJza/7f/MzgjQKiJAkjbfFCXVCACEBsYAQ==" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397052514047,\"observedTimeUnixNano\":1786555400454612062,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}},{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"2 + 2 = 4\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"7576c2384130fece\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "7576c2384130fece" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397052873967,\"observedTimeUnixNano\":1786555400454777045,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}},{\"role\":\"tool\",\"content\":{\"content\":\"[{\\\"toolResult\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"status\\\": \\\"success\\\", \\\"content\\\": [{\\\"text\\\": \\\"4\\\"}]}}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"content\":\"[{\\\"toolUse\\\": {\\\"toolUseId\\\": \\\"tooluse_wZxoTzan8sypzmeYPveFwA\\\", \\\"name\\\": \\\"add_numbers\\\", \\\"input\\\": {\\\"a\\\": 2, \\\"b\\\": 2}}}]\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a1d0f9231c5db33d\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a1d0f9231c5db33d" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQAxgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555397053072269,\"observedTimeUnixNano\":1786555400454894614,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"What is 2+2? Answer concisely.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"2 + 2 = 4\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"67ebf93b-65e3-4127-9e13-483b239f256a\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cabfa3bfe9a7348415c7b507648a8\",\"spanId\":\"a5e83c7e772efdf6\"}" + }, + { + "field": "sessionId", + "value": "67ebf93b-65e3-4127-9e13-483b239f256a" + }, + { + "field": "traceId", + "value": "6a7cabfa3bfe9a7348415c7b507648a8" + }, + { + "field": "spanId", + "value": "a5e83c7e772efdf6" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgagz3ApAAAAAR7TYTwABqfKuvAAAAOSIAEo47b/t/8zML3n/7f/MzgFQJ4zSM9CUJYnIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414476887232,\"observedTimeUnixNano\":1786555419396909949,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"\\\\n You are a helpful assistant. Use tools when appropriate.\\\\n \\\"}]\"}},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"finish_reason\":\"end_turn\",\"message\":\"[{\\\"text\\\": \\\"A primary color is **red**.\\\\n\\\\nThe three primary colors are red, blue, and yellow (in traditional color theory) or red, green, and blue (in the RGB/additive color model used for light).\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"627f6364bb0599fa\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "627f6364bb0599fa" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQAhgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414477366577,\"observedTimeUnixNano\":1786555419397161059,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"system\",\"content\":\"\\n You are a helpful assistant. Use tools when appropriate.\\n \"},{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]},\"output\":{\"messages\":[{\"role\":\"assistant\",\"content\":{\"message\":\"A primary color is **red**.\\n\\nThe three primary colors are red, blue, and yellow (in traditional color theory) or red, green, and blue (in the RGB/additive color model used for light).\\n\",\"finish_reason\":\"end_turn\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"00821fcc8a234519\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "00821fcc8a234519" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQBBgB" + } + ], + [ + { + "field": "@message", + "value": "{\"resource\":{\"attributes\":{\"telemetry.sdk.language\":\"python\",\"telemetry.sdk.name\":\"opentelemetry\",\"telemetry.sdk.version\":\"1.40.0\",\"service.name\":\"asdf_MyAgent.DEFAULT\",\"aws.log.group.names\":\"/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT\",\"aws.log.stream.names\":\"otel-rt-logs\",\"deployment.environment.name\":\"bedrock-agentcore:default\",\"cloud.resource_id\":\"arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q/runtime-endpoint/DEFAULT:DEFAULT\",\"cloud.platform\":\"aws_bedrock_agentcore\",\"cloud.provider\":\"aws\",\"cloud.region\":\"us-west-2\",\"telemetry.auto.version\":\"0.17.0-aws\",\"aws.local.service\":\"asdf_MyAgent.DEFAULT\",\"aws.service.type\":\"gen_ai_agent\"}},\"scope\":{\"name\":\"strands.telemetry.tracer\"},\"timeUnixNano\":1786555414477214324,\"observedTimeUnixNano\":1786555419397054030,\"severityNumber\":9,\"severityText\":\"\",\"body\":{\"input\":{\"messages\":[{\"role\":\"user\",\"content\":{\"content\":\"[{\\\"text\\\": \\\"Name a primary color.\\\"}]\"}}]}},\"attributes\":{\"session.id\":\"7f983b9f-9569-4a4d-bdc2-5c997ff346dd\",\"event.name\":\"strands.telemetry.tracer\"},\"flags\":1,\"traceId\":\"6a7cac0e3fa42bfe5437eef070d1231c\",\"spanId\":\"e90c8e0cb4712af5\"}" + }, + { + "field": "sessionId", + "value": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd" + }, + { + "field": "traceId", + "value": "6a7cac0e3fa42bfe5437eef070d1231c" + }, + { + "field": "spanId", + "value": "e90c8e0cb4712af5" + }, + { + "field": "@ptr", + "value": "CssBCo8BCkw2ODUxOTc3MDg2ODc6L2F3cy9iZWRyb2NrLWFnZW50Y29yZS9ydW50aW1lcy9hc2RmX015QWdlbnQtM3M1YXh2QkM2US1ERUZBVUxUEAAaJGY2ZGY3NDIyLWVlZTQtNDhkNS05NDg3LTE0NTZkYjdkZGE1YiIOCICYk5r/MxDn16zD/zNA0cHR4dszSAASNRoYAgah79P0AAAABj7klGgABqfKwVAAAAdCIAEovO6AuP8zMM3vgLj/MzgFQKovSNk7UMkjIAIQAxgB" + } + ] + ], + "statistics": { + "recordsMatched": 9, + "recordsScanned": 72, + "estimatedRecordsSkipped": 3, + "bytesScanned": 68886, + "estimatedBytesSkipped": 349, + "logGroupsScanned": 1 + }, + "status": "Complete" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json new file mode 100644 index 000000000..4320f369e --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.1623083efa971538.json @@ -0,0 +1,3 @@ +{ + "queryId": "3c802de9-ea3d-4268-ae53-944d121d2618" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json new file mode 100644 index 000000000..cd9387e26 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/StartQueryCommand.8981323eb27080a.json @@ -0,0 +1,3 @@ +{ + "queryId": "cdc73753-869e-4222-b606-a7e5b2e7d9a5" +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json b/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json new file mode 100644 index 000000000..458c13bd7 --- /dev/null +++ b/src/handlers/eval/ondemand/__fixtures__/evaluate.golden.json @@ -0,0 +1,44 @@ +{ + "sessionsRequested": 2, + "sessionsEvaluated": 2, + "results": [ + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "67ebf93b-65e3-4127-9e13-483b239f256a", + "traceId": "6a7cabfa3bfe9a7348415c7b507648a8" + } + }, + "explanation": "The user asked a simple arithmetic question ('What is 2+2?') and requested a concise answer. The assistant's response '2 + 2 = 4' directly and concisely answers the question. The tool output confirmed the answer is 4, and the assistant correctly relayed this information. The response is brief and to the point, matching the user's request for conciseness. This fully satisfies the user's goal with no unnecessary information.", + "value": 0.83, + "label": "Very Helpful", + "tokenUsage": { + "inputTokens": 941, + "outputTokens": 118, + "totalTokens": 1059 + } + }, + { + "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", + "evaluatorId": "Builtin.Helpfulness", + "evaluatorName": "Builtin.Helpfulness", + "context": { + "spanContext": { + "sessionId": "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", + "traceId": "6a7cac0e3fa42bfe5437eef070d1231c" + } + }, + "explanation": "The user's goal was simple: to have a primary color named. The assistant directly answered the question by naming 'red' as a primary color. Beyond just answering the question, the assistant also provided additional context about all three primary colors in both traditional color theory and the RGB model. This extra information is relevant and educational without being overwhelming. The response directly fulfills the user's request and goes a step further by providing useful context about primary colors in general, which anticipates potential follow-up questions or curiosity. This qualifies as 'Above And Beyond' since it answers the question completely and proactively addresses related information the user might find useful.", + "value": 1, + "label": "Above And Beyond", + "tokenUsage": { + "inputTokens": 859, + "outputTokens": 154, + "totalTokens": 1013 + } + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/ondemand/evaluate/index.tsx b/src/handlers/eval/ondemand/evaluate/index.tsx new file mode 100644 index 000000000..b65416509 --- /dev/null +++ b/src/handlers/eval/ondemand/evaluate/index.tsx @@ -0,0 +1,135 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../router"; +import { InputValidationError } from "../../../../errors"; +import { JsonRendererKey } from "../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import type { EvaluationReferenceInput } from "@aws-sdk/client-bedrock-agentcore"; +import { coreOptsFromCtx, parseJsonArrayFlag } from "../../../utils"; +import type { SessionWindow } from "../../types"; + +export const createEvaluateOnDemandHandler = (core: Core, io: AppIO) => + createHandler({ + name: "evaluate", + description: "evaluate existing sessions client-side (synchronous; prints scores)", + flags: [ + flag( + "agent", + "source: harness id or runtime id whose sessions to evaluate", + z.string().optional(), + ), + flag("endpoint", "runtime endpoint qualifier (default DEFAULT)", z.string().optional()), + flag("evaluator", "evaluator id(s) to apply", z.array(z.string()).optional()), + flag( + "lookback-days", + "time filter: evaluate sessions from the last N days", + z.number().optional(), + ), + flag( + "start-time", + "time filter: window start (ISO-8601, with --end-time)", + z.string().optional(), + ), + flag( + "end-time", + "time filter: window end (ISO-8601, with --start-time)", + z.string().optional(), + ), + flag("session-ids", "filter: specific session ids", z.array(z.string()).optional()), + flag( + "trace-id", + "filter: a single trace id (session id is read off the span)", + z.string().optional(), + ), + flag( + "ground-truth", + "ground truth (JSON EvaluationReferenceInput[]; inline, file://, or -)", + z.string().optional(), + ), + ], + handle: async (ctx, flags) => { + if (!flags["agent"]) { + throw new InputValidationError("on-demand requires '--agent'"); + } + if (!flags["evaluator"] || flags["evaluator"].length === 0) { + throw new InputValidationError( + "required option '--evaluator ' not specified", + ); + } + + const window = resolveWindow(flags); + const sessionIds = flags["session-ids"]; + const traceId = flags["trace-id"]; + // On-demand reads sessions client-side, so it needs a concrete source — + // "evaluate everything" is not allowed. + if (!window && !sessionIds?.length && !traceId) { + throw new InputValidationError( + "specify a session source: --session-ids, --trace-id, --lookback-days, or --start-time/--end-time", + ); + } + + const opts = coreOptsFromCtx(ctx); + const traces = await core.eval.getTracesForAgent( + { agent: flags["agent"], endpoint: flags["endpoint"], window, sessionIds, traceId }, + opts, + ); + + // Ground truth is a typed SDK-shape passthrough (identical to batch's handler): + // resolve inline / file:// / -, then hand the array to core verbatim — core + // groups it by session. + const resolver = new SourceResolver({ stdin: io.stdin }); + const groundTruth = parseJsonArrayFlag( + "ground-truth", + await resolver.resolveText("ground-truth", flags["ground-truth"]), + ); + + const result = await core.eval.evaluate( + { traces, evaluatorIds: flags["evaluator"], groundTruth }, + opts, + ); + ctx.require(JsonRendererKey).renderJson(result); + }, + }); + +// resolveWindow validates on-demand's time filter: --lookback-days maps to +// [now - n days, now]; the explicit --start-time/--end-time pair must come together +// with start before end. On-demand owns this rather than reusing batch's resolver: +// batch has no --lookback-days and its window feeds a service-side data source, not +// a client-side Insights query. +function resolveWindow(flags: { + "lookback-days"?: number; + "start-time"?: string; + "end-time"?: string; +}): SessionWindow | undefined { + const lookback = flags["lookback-days"]; + const hasStart = flags["start-time"] !== undefined; + const hasEnd = flags["end-time"] !== undefined; + + if (lookback !== undefined) { + if (hasStart || hasEnd) { + throw new InputValidationError( + "--lookback-days cannot be combined with --start-time/--end-time", + ); + } + if (!Number.isFinite(lookback) || lookback <= 0) { + throw new InputValidationError("--lookback-days must be a positive number"); + } + const endTime = new Date(); + const startTime = new Date(+endTime - lookback * 24 * 60 * 60 * 1000); + return { startTime, endTime }; + } + + if (!hasStart && !hasEnd) return undefined; + if (!hasStart || !hasEnd) { + throw new InputValidationError("--start-time and --end-time must be provided together"); + } + const startTime = new Date(flags["start-time"]!); + const endTime = new Date(flags["end-time"]!); + if (Number.isNaN(+startTime) || Number.isNaN(+endTime)) { + throw new InputValidationError("--start-time and --end-time must be ISO-8601 timestamps"); + } + if (+startTime >= +endTime) { + throw new InputValidationError("--start-time must be before --end-time"); + } + return { startTime, endTime }; +} diff --git a/src/handlers/eval/ondemand/index.tsx b/src/handlers/eval/ondemand/index.tsx new file mode 100644 index 000000000..d46cc38ea --- /dev/null +++ b/src/handlers/eval/ondemand/index.tsx @@ -0,0 +1,13 @@ +import { Router } from "../../../router"; +import type { AppIO } from "../../../io"; +import type { Core } from "../../types"; +import { createHelpDefault } from "../../help"; +import { createEvaluateOnDemandHandler } from "./evaluate"; + +// ondemand groups the synchronous, client-side evaluation commands. It has no TUI +// screen (unlike evaluator/online-eval), so a bare invocation prints help. +export function createOnDemandHandler(core: Core, io: AppIO): Router { + return new Router("ondemand", "evaluate existing sessions synchronously, client-side") + .default(createHelpDefault(io)) + .handler(createEvaluateOnDemandHandler(core, io)); +} diff --git a/src/handlers/eval/ondemand/ondemand.fixture.test.tsx b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx new file mode 100644 index 000000000..f34081c7d --- /dev/null +++ b/src/handlers/eval/ondemand/ondemand.fixture.test.tsx @@ -0,0 +1,88 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; +import { createRootHandler } from "../../index"; + +const REGION = "us-west-2"; +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Record with: RECORD=1 bun test src/handlers/eval/ondemand/ondemand.fixture.test.tsx +// +// This exercises the real seam end to end: parsing → handler → CoreClient → +// getTracesForAgent (GetAgentRuntime + CloudWatch Logs Insights StartQuery / +// GetQueryResults, read from aws/spans and the runtime group) → evaluate (the +// Evaluate data-plane API) → rendered scores. +// +// Determinism: the window is PINNED (not --lookback-days) so the StartQuery input — +// which embeds startTime/endTime epoch seconds — hashes to the same fixture on +// record and replay. --session-ids bounds the fetch to the two sessions recorded +// against the live agent below. +// +// Re-recording needs the agent to still exist AND those sessions' spans to still be +// within CloudWatch retention (they age out). If they've aged out, invoke the agent +// to create fresh sessions, then repoint FIXTURE_SESSION_IDS + the window at them. +const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; +const FIXTURE_SESSION_IDS = [ + "67ebf93b-65e3-4127-9e13-483b239f256a", + "7f983b9f-9569-4a4d-bdc2-5c997ff346dd", +]; +const WINDOW_START = "2026-08-12T00:00:00Z"; +const WINDOW_END = "2026-08-13T00:00:00Z"; + +function createFixtureCore(): CoreClient { + const { createControlClient, createDataClient, createIamClient, createLogsClient } = + fixtureFactories(FIXTURES); + return new CoreClient({ + createControlClient, + createDataClient, + createIamClient, + createLogsClient, + logger: createSilentLogger(), + }); +} + +async function run(args: string[]): Promise { + const io = testIO(); + const root = createRootHandler(createFixtureCore(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", REGION]); + return io.stdout(); +} + +describe("eval ondemand evaluate (fixture-backed)", () => { + test("evaluates the target sessions client-side and prints scores", async () => { + const stdout = await run([ + "eval", + "ondemand", + "evaluate", + "--agent", + FIXTURE_AGENT, + "--session-ids", + ...FIXTURE_SESSION_IDS, + "--start-time", + WINDOW_START, + "--end-time", + WINDOW_END, + "--evaluator", + "Builtin.Helpfulness", + ]); + + matchGolden(FIXTURES, "evaluate.golden.json", stdout); + const result = JSON.parse(stdout); + expect(result.sessionsEvaluated).toBeGreaterThan(0); + expect(result.results.length).toBeGreaterThan(0); + expect(result.results[0].evaluatorId).toBe("Builtin.Helpfulness"); + // Recording polls live CloudWatch Insights (1s between polls) and calls Evaluate + // per session, so it needs well over bun's 5s default; replay is instant. + }, 180_000); +}); diff --git a/src/handlers/eval/ondemand/ondemand.test.tsx b/src/handlers/eval/ondemand/ondemand.test.tsx new file mode 100644 index 000000000..6d7173444 --- /dev/null +++ b/src/handlers/eval/ondemand/ondemand.test.tsx @@ -0,0 +1,178 @@ +import { test, expect, describe } from "bun:test"; +import { createRootHandler } from "../../index"; +import { + createSilentLogger, + TestCoreClient, + testIO, + TestGlobalConfigAccessor, +} from "../../../testing"; +import type { EvaluateResult, SessionTrace } from "../types"; + +// Command-flow tests for `eval ondemand evaluate`, driven through the real root +// handler against a TestCoreClient (no network). These cover the handler's +// orchestration (getTracesForAgent → evaluate), source-arm validation, and the +// local window resolution. The end-to-end SDK path (Insights + Evaluate) is proven +// by the golden fixture suite, which must be recorded against a live account. + +const TRACE: SessionTrace = { + sessionId: "s1", + spans: [{ traceId: "t1", spanId: "sp1" }], + traceIds: ["t1"], + toolCallSpanIds: [], +}; + +const RESULT: EvaluateResult = { + sessionsRequested: 1, + sessionsEvaluated: 1, + results: [{ evaluatorId: "Builtin.Helpfulness", value: 0.9 } as EvaluateResult["results"][number]], +}; + +async function run(args: string[], configure?: (core: TestCoreClient) => void) { + const core = new TestCoreClient(); + core.eval.setGetTracesResponse([TRACE]).setEvaluateResponse(RESULT); + configure?.(core); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + await root.route(["node", "agentcore", ...args, "--region", "us-west-2"]); + return { core, stdout: io.stdout(), stderr: io.stderr() }; +} + +const BASE = [ + "eval", + "ondemand", + "evaluate", + "--agent", + "a-1", + "--evaluator", + "Builtin.Helpfulness", +]; + +describe("eval ondemand command hierarchy", () => { + test("registers evaluate under eval → ondemand", () => { + const io = testIO(); + const root = createRootHandler(new TestCoreClient(), { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + const group = root + .children() + .find((c) => c.name() === "eval") + ?.children() + .find((c) => c.name() === "ondemand"); + expect(group?.children().map((c) => c.name())).toEqual(["evaluate"]); + }); +}); + +describe("eval ondemand evaluate validation", () => { + test("requires --agent", async () => { + await expect( + run([ + "eval", + "ondemand", + "evaluate", + "--evaluator", + "Builtin.Helpfulness", + "--session-ids", + "s1", + ]), + ).rejects.toThrow(/--agent/); + }); + + test("requires --evaluator", async () => { + await expect( + run(["eval", "ondemand", "evaluate", "--agent", "a-1", "--session-ids", "s1"]), + ).rejects.toThrow(/--evaluator/); + }); + + test("rejects an empty session source", async () => { + await expect(run(BASE)).rejects.toThrow(/session source/); + }); + + test("rejects --lookback-days combined with an explicit window", async () => { + await expect( + run([ + ...BASE, + "--lookback-days", + "7", + "--start-time", + "2026-01-01T00:00:00Z", + "--end-time", + "2026-01-02T00:00:00Z", + ]), + ).rejects.toThrow(/cannot be combined/); + }); + + test("rejects a half-open explicit window", async () => { + await expect(run([...BASE, "--start-time", "2026-01-01T00:00:00Z"])).rejects.toThrow( + /together/, + ); + }); + + test("rejects start-time not before end-time", async () => { + await expect( + run([...BASE, "--start-time", "2026-01-02T00:00:00Z", "--end-time", "2026-01-01T00:00:00Z"]), + ).rejects.toThrow(/before/); + }); +}); + +describe("eval ondemand evaluate orchestration", () => { + test("session-ids arm: fetches traces then evaluates them, rendering the result", async () => { + const { core, stdout } = await run([...BASE, "--session-ids", "s1", "s2"]); + + expect(JSON.parse(stdout)).toEqual(RESULT); + + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch?.args[0]).toMatchObject({ + agent: "a-1", + sessionIds: ["s1", "s2"], + window: undefined, + }); + + // evaluate receives exactly the traces getTracesForAgent returned. + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ + traces: [TRACE], + evaluatorIds: ["Builtin.Helpfulness"], + }); + // Order: fetch precedes evaluate. + expect(core.eval.calls.map((c) => c.method)).toEqual(["getTracesForAgent", "evaluate"]); + }); + + test("--trace-id alone is a valid source and is passed to the fetch", async () => { + const { core } = await run([...BASE, "--trace-id", "t1"]); + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch?.args[0]).toMatchObject({ traceId: "t1" }); + }); + + test("--lookback-days resolves to a now-N-days window (start before end)", async () => { + const { core } = await run([...BASE, "--lookback-days", "7"]); + const fetch = core.eval.calls.find((c) => c.method === "getTracesForAgent"); + expect(fetch).toBeDefined(); + const input = fetch!.args[0] as { window?: { startTime: Date; endTime: Date } }; + expect(input.window).toBeDefined(); + const window = input.window!; + expect(+window.startTime).toBeLessThan(+window.endTime); + const spanDays = (+window.endTime - +window.startTime) / (24 * 60 * 60 * 1000); + expect(spanDays).toBeCloseTo(7, 5); + }); + + test("ground-truth is parsed and passed to evaluate verbatim", async () => { + const groundTruth = [ + { context: { spanContext: { sessionId: "s1" } }, expectedResponse: { text: "hi" } }, + ]; + const { core } = await run([ + ...BASE, + "--session-ids", + "s1", + "--ground-truth", + JSON.stringify(groundTruth), + ]); + const evaluate = core.eval.calls.find((c) => c.method === "evaluate"); + expect(evaluate?.args[0]).toMatchObject({ groundTruth }); + }); +}); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 1ef70c6a8..07687048e 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -34,6 +34,8 @@ import type { ListBatchEvaluationsResponse, StartBatchEvaluationResponse, SessionMetadataShape, + EvaluationReferenceInput, + EvaluationResultContent, DataSourceConfig as DataPlaneDataSourceConfig, } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; @@ -203,6 +205,84 @@ export type StartBatchEvaluationInput = { kmsKeyArn?: string; }; +// SimulateInput is the CLI-facing shape for `batch-evaluation simulate` — dataset +// replay + batch grade. SDK-native fields only (no RuntimeInvokeRequest/EvaluateInput +// leak): Core invokes each scenario against the runtime, then submits a batch +// evaluation scoped to the sessions it created. Invoke fields mirror `runtime invoke`. +export type SimulateInput = { + runtimeId: string; + qualifier?: string; + payloadTemplate: string; // e.g. {"prompt":"{input}"} — {input} is the scenario's turn input + headers?: [string, string][]; + bearerToken?: string; + // simulate always creates a fresh session per scenario. Reusing one session + // across scenarios interleaves unrelated turns and collides ground-truth keys. + userId?: string; + dataset: string; // local JSONL path or a dataset id + datasetVersion?: string; + evaluatorIds: string[]; + name: string; + description?: string; + kmsKeyArn?: string; +}; + +// SimulateResult reports the submitted job plus how many scenarios were actually +// invoked vs dropped (a failed invoke is skipped, not fatal, unless all fail). +export type SimulateResult = { + batchEvaluationId?: string; + status?: string; + scenariosInvoked: number; + scenariosFailed: number; +}; + +// SpanRecord is one OTel span/log document — the parsed `@message` JSON of a +// CloudWatch Logs Insights result row. Left open (arbitrary JSON) because it is +// handed to the Evaluate API's `sessionSpans` verbatim; the CLI only reads a few +// well-known fields off it (traceId, spanId, attributes) to route evaluators. +export type SpanRecord = Record; + +// SessionTrace is one session's gathered telemetry, grouped client-side. Neutral +// by design — no Evaluate coupling — so getTracesForAgent stays reusable and +// EvalClient.evaluate owns the mapping into the Evaluate request shape. +export type SessionTrace = { + sessionId: string; // read from attributes.session.id + spans: SpanRecord[]; // full OTel span JSON (@message) + traceIds: string[]; // for TRACE-level evaluators (evaluationTarget.traceIds) + toolCallSpanIds: string[]; // for TOOL_CALL-level evaluators (evaluationTarget.spanIds) +}; + +// GetTracesInput selects which sessions' traces to read for one agent. `sessionIds` +// and `traceId` are independent, optional, AND-ed query filters; with neither, the +// `window` bounds discovery. `window` unset ⇒ the client defaults to now−7d. +export type GetTracesInput = { + agent: string; + endpoint?: string; + window?: SessionWindow; + sessionIds?: string[]; + traceId?: string; +}; + +// EvaluateInput is the CLI-facing shape for the synchronous Evaluate path. +// `groundTruth` is the SDK-native array, passed verbatim; EvalClient groups it by +// session (context.spanContext.sessionId) and attaches per session. +export type EvaluateInput = { + traces: SessionTrace[]; + evaluatorIds: string[]; + groundTruth?: EvaluationReferenceInput[]; +}; + +// EvaluateResult returns the raw Evaluate API results across all evaluators and +// sessions (each carries its own evaluatorId + span context). No aggregation — the +// caller renders the raw scores. The two counts are distinct on purpose: +// `sessionsRequested` is how many gathered sessions were handed to Evaluate; +// `sessionsEvaluated` is how many actually produced results (a TRACE/TOOL_CALL +// session with no matching ids is requested but not evaluated). +export type EvaluateResult = { + sessionsRequested: number; + sessionsEvaluated: number; + results: EvaluationResultContent[]; +}; + // DatasetUpdateResult reports what reconciling a DRAFT against a local file // changed. The counts are examples, not requests: each phase is batched to the // service's per-request limit. @@ -270,6 +350,24 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; + // getTracesForAgent resolves the agent to its runtime log group and reads the + // target sessions' traces client-side (CloudWatch Logs Insights), grouped by + // session. Returns neutral SessionTrace records — the ondemand handler hands + // them to evaluate. Kept off the Evaluate path so the same fetch is reusable. + getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise; + // evaluate runs evaluators synchronously over already-gathered traces via the + // Evaluate API and returns per-session scores. No job, no CloudWatch — the + // trace read happened in getTracesForAgent. + evaluate(input: EvaluateInput, options: CoreOptions): Promise; + // simulate replays a dataset against the runtime (invoke per scenario, client-side) + // then submits a batch evaluation over the sessions it created. No dataset API + // exists service-side, so the CLI creates the sessions and the service grades them. + simulate( + input: SimulateInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; + createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions, diff --git a/src/io/index.ts b/src/io/index.ts index e93169768..7fb34175b 100644 --- a/src/io/index.ts +++ b/src/io/index.ts @@ -26,6 +26,7 @@ export { type JsonValue, } from "./jsonl"; export { SourceResolver, type SourceResolverConfig } from "./source"; +export { renderJsonTemplate } from "./template"; export { classifyStreamingResponse, writeStreamingResponse, diff --git a/src/io/template.test.ts b/src/io/template.test.ts new file mode 100644 index 000000000..6be6da2de --- /dev/null +++ b/src/io/template.test.ts @@ -0,0 +1,32 @@ +import { test, expect, describe } from "bun:test"; +import { renderJsonTemplate } from "./template"; + +const decode = (bytes: Uint8Array) => new TextDecoder().decode(bytes); + +describe("renderJsonTemplate", () => { + test("substitutes {input} inside a string value", () => { + const out = decode(renderJsonTemplate('{"prompt":"{input}"}', { input: "hello" })); + expect(JSON.parse(out)).toEqual({ prompt: "hello" }); + }); + + test("JSON-escapes substituted values (quotes, newlines don't break the payload)", () => { + const out = renderJsonTemplate('{"prompt":"{input}"}', { input: 'a "quote"\nline' }); + expect(JSON.parse(decode(out))).toEqual({ prompt: 'a "quote"\nline' }); + }); + + test("substitutes nested + array positions", () => { + const out = renderJsonTemplate('{"messages":[{"role":"user","content":"{input}"}]}', { + input: "hi", + }); + expect(JSON.parse(decode(out))).toEqual({ messages: [{ role: "user", content: "hi" }] }); + }); + + test("supports arbitrary keys, leaves unknown placeholders intact", () => { + const out = renderJsonTemplate('{"m":"{model}","p":"{input}"}', { input: "x" }); + expect(JSON.parse(decode(out))).toEqual({ m: "{model}", p: "x" }); + }); + + test("rejects invalid JSON template", () => { + expect(() => renderJsonTemplate("{not json", { input: "x" })).toThrow(/valid JSON/); + }); +}); diff --git a/src/io/template.ts b/src/io/template.ts new file mode 100644 index 000000000..e7d12cca8 --- /dev/null +++ b/src/io/template.ts @@ -0,0 +1,33 @@ +import { InputValidationError } from "../errors"; + +// renderJsonTemplate substitutes `{key}` placeholders inside the string values of a +// JSON template and returns the encoded bytes. General on purpose: `simulate` uses +// `{ input }`, but any `{model}`/`{sessionId}` a future caller adds works the same. +// Parsing the template first (rather than string-replacing raw) keeps the result +// valid JSON regardless of quotes/newlines in the substituted values. +export function renderJsonTemplate( + template: string, + values: Record, + flagName = "payload-template", +): Uint8Array { + let parsed: unknown; + try { + parsed = JSON.parse(template); + } catch { + throw new InputValidationError(`--${flagName} must be valid JSON`); + } + return new TextEncoder().encode(JSON.stringify(substitute(parsed, values))); +} + +function substitute(value: unknown, values: Record): unknown { + if (typeof value === "string") { + return value.replace(/\{(\w+)\}/g, (match, key) => values[key] ?? match); + } + if (Array.isArray(value)) return value.map((item) => substitute(item, values)); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, substitute(item, values)]), + ); + } + return value; +} diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index c39f6a278..63392c571 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -131,8 +131,14 @@ import type { CreateOnlineEvalInput, DatasetUpdateResult, DatasetUpdateProgressEvent, + EvaluateInput, + EvaluateResult, + SimulateInput, + SimulateResult, GetBatchEvaluationResult, + GetTracesInput, LlmAsAJudgeUpdate, + SessionTrace, StartBatchEvaluationInput, UpdateConfigurationBundleInput, UpdateOnlineEvalInput, @@ -1393,6 +1399,18 @@ export class TestEvalClient implements CoreEvalClient { private batchEvalResults: BatchEvaluationResultEntry[] = []; private batchEvalResultsError?: unknown; private startBatchEvalResponse: StartBatchEvaluationResponse = DEFAULT_START_BATCH_EVAL_RESPONSE; + private getTracesResponse: SessionTrace[] = []; + private evaluateResponse: EvaluateResult = { + sessionsRequested: 0, + sessionsEvaluated: 0, + results: [], + }; + private simulateResponse: SimulateResult = { + batchEvaluationId: "batch-eval-test", + status: "RUNNING", + scenariosInvoked: 0, + scenariosFailed: 0, + }; private error?: Error; // setListResponse sets what listEvaluators resolves to (when not erroring). @@ -1687,6 +1705,47 @@ export class TestEvalClient implements CoreEvalClient { return this.startBatchEvalResponse; } + // setGetTracesResponse sets what getTracesForAgent resolves to (when not + // erroring). + setGetTracesResponse(traces: SessionTrace[]): this { + this.getTracesResponse = traces; + return this; + } + + // setEvaluateResponse sets what evaluate resolves to (when not erroring). + setEvaluateResponse(response: EvaluateResult): this { + this.evaluateResponse = response; + return this; + } + + async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { + this.calls.push({ method: "getTracesForAgent", args: [input, options] }); + if (this.error) throw this.error; + return this.getTracesResponse; + } + + async evaluate(input: EvaluateInput, options: CoreOptions): Promise { + this.calls.push({ method: "evaluate", args: [input, options] }); + if (this.error) throw this.error; + return this.evaluateResponse; + } + + // setSimulateResponse sets what simulate resolves to (when not erroring). + setSimulateResponse(response: SimulateResult): this { + this.simulateResponse = response; + return this; + } + + async simulate( + input: SimulateInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + this.calls.push({ method: "simulate", args: [input, options, signal] }); + if (this.error) throw this.error; + return this.simulateResponse; + } + async createOnlineEvaluationConfig( input: CreateOnlineEvalInput, options: CoreOptions,