Skip to content

Commit 4a5d3fb

Browse files
Qardclaude
Qard
and
claude
committed
Add Mastra auto-instrumentation via ObservabilityExporter
Replaces the chunk-AST instrumentation with a Braintrust ObservabilityExporter that the loader auto-installs into every `new Mastra(...)`. Survives Mastra's content-hashed chunk renames release-to-release because the loader only patches the stable `dist/index.{js,cjs}` entrypoints — chunk paths are read out of the original source at load time, not pinned in a version table. Two integration paths: - Auto: under `node --import braintrust/hook.mjs`, the loader rewrites Mastra's entry into a Proxy-wrapped class that auto-constructs an `Observability` instance if none was provided, and appends a Proxy wrap to `@mastra/observability`'s entry so the constructor injects our exporter into every config. - Manual: `import { BraintrustObservabilityExporter } from "braintrust";` and pass it to `new Mastra({ observability: new Observability({...}) })`. Requires `@mastra/core >= 1.20.0` for the auto path; manual integration works on any Mastra version that accepts an `ObservabilityExporter`. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 50a79f0 commit 4a5d3fb

23 files changed

Lines changed: 4540 additions & 26 deletions

‎.changeset/five-pens-dance.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
"braintrust": minor
3+
---
4+
5+
feat(mastra): auto-instrument Mastra via its native `ObservabilityExporter`
6+
7+
Replaces the chunk-AST instrumentation with a Braintrust `ObservabilityExporter` that the loader auto-installs into every `new Mastra(...)`. Survives Mastra's content-hashed chunk renames release-to-release because the loader only touches the stable `dist/mastra/index.{js,cjs}` entry point.
8+
9+
Two integration paths:
10+
11+
- **Auto** (default with `node --import braintrust/hook.mjs`): no user code change, the loader wraps `Mastra` to call `defaultInstance.registerExporter(...)` after construction. Requires the user to enable observability via `new Mastra({ observability: new Observability({ ... }) })`.
12+
- **Manual**: `import { BraintrustObservabilityExporter } from "braintrust";` and pass it via `new Mastra({ observability: new Observability({ configs: { default: { exporters: [new BraintrustObservabilityExporter()] } } }) })`.
13+
14+
Requires `@mastra/core >= 1.20.0` for the auto path (the version that added `Mastra.prototype.registerExporter`); older versions silently no-op. Manual integration works on any Mastra version that accepts an `ObservabilityExporter`.

‎e2e/config/pr-comment-scenarios.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@
6868
"metadataScenario": "genkit-instrumentation",
6969
"variants": [{ "variantKey": "genkit-v1-33-0", "label": "v1.33.0" }]
7070
},
71+
{
72+
"scenarioDirName": "mastra-instrumentation",
73+
"label": "Mastra Instrumentation",
74+
"metadataScenario": "mastra-instrumentation",
75+
"variants": [{ "variantKey": "mastra-v1260", "label": "v1.26.0" }]
76+
},
7177
{
7278
"scenarioDirName": "groq-instrumentation",
7379
"label": "Groq Instrumentation",

‎e2e/helpers/normalize.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,12 @@ const TEMP_HELPER_PATH_REGEX = /\/e2e\/\.bt-tmp\/[^/\s)]+\/helpers\/?/g;
9797
const PROVIDER_HELPER_CALLER_REGEX = /^<repo>\/e2e\/helpers\/.+-scenario\.mjs$/;
9898
const ANTHROPIC_MESSAGE_STREAM_PATH_REGEX =
9999
/([/\\]node_modules[/\\]\.pnpm[/\\]@anthropic-ai\+sdk@[^/\\\s)]+[/\\]node_modules[/\\]@anthropic-ai[/\\]sdk[/\\])(?:src[/\\]lib[/\\]MessageStream\.ts|lib[/\\]MessageStream\.js)/g;
100+
// tsup's `splitting: true` for our own `dist/` emits content-hashed chunk
101+
// files (e.g. `<repo>/js/dist/chunk-7DWPOXBX.mjs`) whose names change any
102+
// time the bundle graph changes. Normalize them to a stable placeholder so
103+
// stack traces in error snapshots don't churn on unrelated bundle splits.
104+
const SDK_CHUNK_PATH_REGEX =
105+
/(<repo>\/js\/dist\/)chunk-[A-Z0-9]+(\.(?:c?js|cjs|mjs))/g;
100106
const ANTHROPIC_PNPM_VERSION_REGEX =
101107
/([/\\]\.pnpm[/\\]@anthropic-ai\+sdk@)[^/\\\s)]+/g;
102108

@@ -153,6 +159,7 @@ function normalizeStackLikeString(value: string): string {
153159
normalized = normalized.replace(REPO_PATH_REGEX, (_, suffix: string) => {
154160
return `<repo>${suffix.replace(/\\/g, "/")}`;
155161
});
162+
normalized = normalized.replace(SDK_CHUNK_PATH_REGEX, "$1index$2");
156163
normalized = normalized.replace(
157164
/(<repo>(?:\/(?:e2e|js)\/[^:\s)\n]+)):\d+:\d+/g,
158165
"$1:0:0",

‎e2e/scenarios/mastra-instrumentation/__snapshots__/mastra-v1260.log-payloads.json‎

Lines changed: 266 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,266 @@
1+
[
2+
{
3+
"metadata": {
4+
"scenario": "mastra-instrumentation"
5+
},
6+
"metric_keys": [],
7+
"name": "mastra-instrumentation-root",
8+
"type": "task"
9+
},
10+
{
11+
"metadata": {
12+
"operation": "generate"
13+
},
14+
"metric_keys": [],
15+
"name": "mastra-agent-generate-operation",
16+
"type": null
17+
},
18+
{
19+
"input": "What is the weather in Paris?",
20+
"metadata": {
21+
"entity_id": "weather-agent",
22+
"entity_name": "Weather Agent",
23+
"entity_type": "agent"
24+
},
25+
"metric_keys": [],
26+
"name": "agent run: 'weather-agent'",
27+
"output": {
28+
"files": [],
29+
"text": "The forecast is sunny."
30+
},
31+
"type": "task"
32+
},
33+
{
34+
"input": {
35+
"messages": [
36+
{
37+
"content": "Answer weather questions with the provided mock forecast.",
38+
"role": "system"
39+
},
40+
{
41+
"content": [
42+
{
43+
"providerOptions": {
44+
"mastra": {
45+
"createdAt": 0
46+
}
47+
},
48+
"text": "What is the weather in Paris?",
49+
"type": "text"
50+
}
51+
],
52+
"role": "user"
53+
}
54+
]
55+
},
56+
"metadata": {
57+
"entity_id": "weather-agent",
58+
"entity_name": "Weather Agent",
59+
"entity_type": "agent",
60+
"model": "mock-model-id",
61+
"provider": "mock-provider"
62+
},
63+
"metric_keys": [
64+
"completion_tokens",
65+
"prompt_tokens",
66+
"tokens"
67+
],
68+
"name": "llm: 'mock-model-id'",
69+
"output": {
70+
"files": [],
71+
"reasoning": [],
72+
"sources": [],
73+
"text": "The forecast is sunny.",
74+
"warnings": []
75+
},
76+
"type": "llm"
77+
},
78+
{
79+
"input": {},
80+
"metadata": {
81+
"entity_id": "weather-agent",
82+
"entity_name": "Weather Agent",
83+
"entity_type": "agent"
84+
},
85+
"metric_keys": [
86+
"completion_tokens",
87+
"prompt_tokens",
88+
"tokens"
89+
],
90+
"name": "step: 0",
91+
"output": {
92+
"text": "The forecast is sunny.",
93+
"toolCalls": []
94+
},
95+
"type": "llm"
96+
},
97+
{
98+
"metadata": {
99+
"entity_id": "weather-agent",
100+
"entity_name": "Weather Agent",
101+
"entity_type": "agent"
102+
},
103+
"metric_keys": [],
104+
"name": "chunk: 'text'",
105+
"output": {
106+
"text": "The forecast is sunny."
107+
},
108+
"type": "llm"
109+
},
110+
{
111+
"metadata": {
112+
"operation": "stream"
113+
},
114+
"metric_keys": [],
115+
"name": "mastra-agent-stream-operation",
116+
"type": null
117+
},
118+
{
119+
"input": "Stream the Paris forecast.",
120+
"metadata": {
121+
"entity_id": "weather-agent",
122+
"entity_name": "Weather Agent",
123+
"entity_type": "agent"
124+
},
125+
"metric_keys": [],
126+
"name": "agent run: 'weather-agent'",
127+
"output": {
128+
"files": [],
129+
"text": "The forecast is sunny."
130+
},
131+
"type": "task"
132+
},
133+
{
134+
"input": {
135+
"messages": [
136+
{
137+
"content": "Answer weather questions with the provided mock forecast.",
138+
"role": "system"
139+
},
140+
{
141+
"content": [
142+
{
143+
"providerOptions": {
144+
"mastra": {
145+
"createdAt": 0
146+
}
147+
},
148+
"text": "Stream the Paris forecast.",
149+
"type": "text"
150+
}
151+
],
152+
"role": "user"
153+
}
154+
]
155+
},
156+
"metadata": {
157+
"entity_id": "weather-agent",
158+
"entity_name": "Weather Agent",
159+
"entity_type": "agent",
160+
"model": "mock-model-id",
161+
"provider": "mock-provider"
162+
},
163+
"metric_keys": [
164+
"completion_tokens",
165+
"prompt_tokens",
166+
"tokens"
167+
],
168+
"name": "llm: 'mock-model-id'",
169+
"output": {
170+
"files": [],
171+
"reasoning": [],
172+
"sources": [],
173+
"text": "The forecast is sunny.",
174+
"warnings": []
175+
},
176+
"type": "llm"
177+
},
178+
{
179+
"input": {},
180+
"metadata": {
181+
"entity_id": "weather-agent",
182+
"entity_name": "Weather Agent",
183+
"entity_type": "agent"
184+
},
185+
"metric_keys": [
186+
"completion_tokens",
187+
"prompt_tokens",
188+
"tokens"
189+
],
190+
"name": "step: 0",
191+
"output": {
192+
"text": "The forecast is sunny.",
193+
"toolCalls": []
194+
},
195+
"type": "llm"
196+
},
197+
{
198+
"metadata": {
199+
"entity_id": "weather-agent",
200+
"entity_name": "Weather Agent",
201+
"entity_type": "agent"
202+
},
203+
"metric_keys": [],
204+
"name": "chunk: 'text'",
205+
"output": {
206+
"text": "The forecast is sunny."
207+
},
208+
"type": "llm"
209+
},
210+
{
211+
"metadata": {
212+
"operation": "tool"
213+
},
214+
"metric_keys": [],
215+
"name": "mastra-tool-operation",
216+
"type": null
217+
},
218+
{
219+
"metadata": {
220+
"operation": "workflow"
221+
},
222+
"metric_keys": [],
223+
"name": "mastra-workflow-operation",
224+
"type": null
225+
},
226+
{
227+
"input": {
228+
"city": "Berlin"
229+
},
230+
"metadata": {
231+
"entity_id": "travel-flow",
232+
"entity_name": "travel-flow",
233+
"entity_type": "workflow_run"
234+
},
235+
"metric_keys": [],
236+
"name": "workflow run: 'travel-flow'",
237+
"output": {
238+
"forecast": "Sunny in Berlin"
239+
},
240+
"type": "task"
241+
},
242+
{
243+
"input": {
244+
"city": "Berlin"
245+
},
246+
"metadata": {
247+
"entity_id": "lookup-step",
248+
"entity_name": "travel-flow",
249+
"entity_type": "workflow_step"
250+
},
251+
"metric_keys": [],
252+
"name": "workflow step: 'lookup-step'",
253+
"output": {
254+
"forecast": "Sunny in Berlin"
255+
},
256+
"type": "function"
257+
},
258+
{
259+
"metadata": {
260+
"scenario": "mastra-instrumentation"
261+
},
262+
"metric_keys": [],
263+
"name": "mastra-instrumentation-root",
264+
"type": "task"
265+
}
266+
]

0 commit comments

Comments
 (0)