feat(dev): collect local OTEL traces in project dev

package.json

@@ -7,6 +7,9 @@
     "agentcore": "./dist/index.js"
   },
   "main": "./dist/index.js",
+  "engines": {
+    "node": ">=20.12.0"
+  },
   "files": [
     "dist"
   ],
@@ -53,6 +56,7 @@
     "@aws-sdk/client-bedrock-agentcore-control": "^3.1102.0",
     "@aws-sdk/client-cloudwatch-logs": "^3.1092.0",
     "@aws-sdk/client-iam": "^3.1080.0",
+    "@opentelemetry/otlp-transformer": "0.213.0",
     "@smithy/core": "3.29.3",
     "@tanstack/react-query": "^5.101.2",
     "cli-truncate": "^6.1.1",

src/assets/templates/shared/env.local.template

@@ -1,7 +1,11 @@
 # Environment variables for local development.
-# `agentcore dev` loads this file into your agent's process. Values here
-# override anything the CLI injects. This file is gitignored — keep secrets
-# out of version control, but they are safe here.
+# `agentcore project dev` loads this file into your agent's process. Values here
+# override injected values except PORT, FASTMCP_PORT, and LOCAL_DEV, which the
+# CLI owns. While trace collection is on (the default), the CLI also owns the
+# OTEL_* and AGENT_OBSERVABILITY_ENABLED variables so traces reach its local
+# collector — pass --no-traces (or set instrumentation.enableOtel to false in
+# agentcore.json) to disable collection and set your own.
+# This file is gitignored — keep secrets out of version control.
 #
 # Example:
 # MY_API_KEY=...

src/core/dev/codezip.test.ts

@@ -1,10 +1,10 @@
 import { afterEach, describe, expect, test } from "bun:test";
-import { mkdir, mkdtemp, rm } from "node:fs/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import type { ProjectRuntime } from "../project/schema";
 import type { DevEvent, DevServerInput } from "../../handlers/project/dev/types";
-import type { ProcessEvent, ProcessStreamer, StreamProcessOptions } from "../../io";
+import type { ProcessEvent, ProcessRunner, ProcessStreamer, StreamProcessOptions } from "../../io";
 import { CodeZipDevRunner } from "./codezip";
 
 type ProcessCall = {
@@ -43,15 +43,22 @@ async function projectRoot(withNodeModules = false): Promise<string> {
   return root;
 }
 
-function harness(output: ProcessEvent[] = []) {
+function harness(output: ProcessEvent[] = [], probe: { dir?: string; fail?: boolean } = {}) {
   const calls: ProcessCall[] = [];
+  const probeCalls: string[][] = [];
   const fakeStreamProcess: ProcessStreamer = async function* (command, options) {
     calls.push({ command, options });
     yield* output;
   };
+  const fakeRunProcess: ProcessRunner = async (command, options) => {
+    probeCalls.push(command);
+    if (probe.fail) throw new Error("probe failed");
+    options.onOutput?.(`${probe.dir ?? ""}\n`);
+  };
   return {
     calls,
-    runner: new CodeZipDevRunner({ streamProcess: fakeStreamProcess }),
+    probeCalls,
+    runner: new CodeZipDevRunner({ streamProcess: fakeStreamProcess, runProcess: fakeRunProcess }),
   };
 }
 
@@ -153,3 +160,69 @@ describe("CodeZipDevRunner", () => {
     ]);
   });
 });
+
+describe("CodeZipDevRunner OTEL instrumentation", () => {
+  async function sitecustomizeDir(): Promise<string> {
+    const directory = await mkdtemp(join(tmpdir(), "otel-site-"));
+    tempDirectories.push(directory);
+    await writeFile(join(directory, "sitecustomize.py"), "");
+    return directory;
+  }
+
+  function otelInput(root: string, extraEnv: Record<string, string> = {}): DevServerInput {
+    const base = input(root, runtime());
+    return {
+      ...base,
+      env: { ...base.env, OTEL_EXPORTER_OTLP_ENDPOINT: "http://127.0.0.1:4318", ...extraEnv },
+    };
+  }
+
+  test("prepends the sitecustomize directory to PYTHONPATH when instrumentation is installed", async () => {
+    const root = await projectRoot();
+    const directory = await sitecustomizeDir();
+    const { calls, probeCalls, runner } = harness([], { dir: directory });
+
+    await collect(runner.run(otelInput(root)));
+
+    expect(probeCalls[0]?.slice(0, 4)).toEqual(["uv", "run", "python", "-c"]);
+    expect(calls[0]?.options.env?.PYTHONPATH).toBe(directory);
+  });
+
+  test("preserves an existing PYTHONPATH", async () => {
+    const root = await projectRoot();
+    const directory = await sitecustomizeDir();
+    const { calls, runner } = harness([], { dir: directory });
+
+    await collect(runner.run(otelInput(root, { PYTHONPATH: "/existing" })));
+
+    expect(calls[0]?.options.env?.PYTHONPATH).toBe(`${directory}:/existing`);
+  });
+
+  test("does not probe without an OTEL endpoint or for Node entrypoints", async () => {
+    const root = await projectRoot(true);
+    const { probeCalls, runner } = harness();
+
+    await collect(runner.run(input(root, runtime())));
+    await collect(runner.run({ ...otelInput(root), runtime: runtime({ entrypoint: "index.js" }) }));
+
+    expect(probeCalls).toEqual([]);
+  });
+
+  test.each([
+    ["probe failure", { fail: true }],
+    ["missing sitecustomize.py", { dir: "/nonexistent" }],
+  ] as const)("warns and starts untraced on %s", async (_case, probe) => {
+    const root = await projectRoot();
+    const { calls, probeCalls, runner } = harness([], probe);
+
+    const events = await collect(runner.run(otelInput(root)));
+
+    expect(probeCalls).toHaveLength(1);
+    expect(calls).toHaveLength(1);
+    expect(calls[0]?.options.env?.PYTHONPATH).toBeUndefined();
+    expect(events).toContainEqual({
+      type: "status",
+      message: expect.stringContaining("traces will not be collected"),
+    });
+  });
+});

src/core/dev/codezip.ts

@@ -1,18 +1,27 @@
 import { existsSync } from "node:fs";
-import { join } from "node:path";
+import { delimiter, join } from "node:path";
 import { InputValidationError } from "../../errors";
 import type { DevEvent, DevRunner, DevServerInput } from "../../handlers/project/dev/types";
-import { streamProcess, type ProcessStreamer, type StreamProcessOptions } from "../../io";
+import {
+  runProcess,
+  streamProcess,
+  type ProcessRunner,
+  type ProcessStreamer,
+  type StreamProcessOptions,
+} from "../../io";
 
 type CodeZipDevRunnerConfig = {
   streamProcess?: ProcessStreamer;
+  runProcess?: ProcessRunner;
 };
 
 export class CodeZipDevRunner implements DevRunner {
   private readonly streamProcess: ProcessStreamer;
+  private readonly runProcess: ProcessRunner;
 
   constructor(config: CodeZipDevRunnerConfig = {}) {
     this.streamProcess = config.streamProcess ?? streamProcess;
+    this.runProcess = config.runProcess ?? runProcess;
   }
 
   public async *run(input: DevServerInput): AsyncGenerator<DevEvent, void> {
@@ -33,8 +42,48 @@ export class CodeZipDevRunner implements DevRunner {
 
     yield { type: "status", message: "Starting development server" };
     const serverProcess = commandForRuntime(entrypoint!, directory, input);
+    if (entrypoint!.endsWith(".py") && input.env?.OTEL_EXPORTER_OTLP_ENDPOINT) {
+      const sitecustomizeDir = await this.findOtelSitecustomizeDir(directory);
+      if (sitecustomizeDir) {
+        const existing = serverProcess.options.env?.PYTHONPATH;
+        serverProcess.options.env = {
+          ...serverProcess.options.env,
+          PYTHONPATH: existing ? `${sitecustomizeDir}${delimiter}${existing}` : sitecustomizeDir,
+        };
+      } else {
+        yield {
+          type: "status",
+          message:
+            "OTEL auto-instrumentation is not installed in the agent environment; traces will not be collected. Add aws-opentelemetry-distro to the agent's dependencies to enable them.",
+        };
+      }
+    }
     yield* this.streamProcess(serverProcess.command, serverProcess.options);
   }
+
+  /**
+   * Locate the auto-instrumentation sitecustomize.py directory inside the agent's
+   * uv environment. Prepending it to PYTHONPATH instruments every Python process —
+   * an `opentelemetry-instrument` wrapper would only instrument uvicorn's reloader
+   * parent, leaving the re-spawned worker processes untraced.
+   */
+  private async findOtelSitecustomizeDir(directory: string): Promise<string | undefined> {
+    const output: string[] = [];
+    const probe =
+      "import opentelemetry.instrumentation.auto_instrumentation as m; import os; print(os.path.dirname(m.__file__))";
+    try {
+      await this.runProcess(["uv", "run", "python", "-c", probe], {
+        cwd: directory,
+        onOutput: (chunk) => output.push(chunk),
+      });
+    } catch {
+      return undefined;
+    }
+    const sitecustomizeDir = output.join("").trim().split("\n").at(-1)?.trim();
+    if (!sitecustomizeDir || !existsSync(join(sitecustomizeDir, "sitecustomize.py")))
+      return undefined;
+    return sitecustomizeDir;
+  }
 }
 
 function commandForRuntime(