From 8ee214ab9719fd6aa6e52214c730c5c61123c136 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 13 May 2026 11:44:38 -0700 Subject: [PATCH 1/4] feat(observability): export Trigger.dev telemetry to Grafana Cloud OTLP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire OTLP HTTP exporters for traces, logs, and metrics from the Trigger.dev runtime to Grafana Cloud. Auth uses Basic with instance ID and API token. Gated behind GRAFANA_OTLP_ENDPOINT, GRAFANA_INSTANCE_ID, and GRAFANA_API_TOKEN — all three must be set together or all unset; partial config throws at startup. --- apps/sim/lib/core/config/env.ts | 3 +++ apps/sim/package.json | 2 ++ apps/sim/trigger.config.ts | 30 ++++++++++++++++++++++++++++++ bun.lock | 2 ++ 4 files changed, 37 insertions(+) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 1c391dfff53..5b532af563f 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -157,6 +157,9 @@ export const env = createEnv({ LOG_LEVEL: z.enum(['DEBUG', 'INFO', 'WARN', 'ERROR']).optional(), // Minimum log level to display (defaults to ERROR in production, DEBUG in development) PROFOUND_API_KEY: z.string().min(1).optional(), // Profound analytics API key PROFOUND_ENDPOINT: z.string().url().optional(), // Profound analytics endpoint + GRAFANA_OTLP_ENDPOINT: z.string().url().optional(), // Grafana Cloud OTLP HTTP gateway base URL (e.g., https://otlp-gateway-prod-us-east-0.grafana.net/otlp). Trigger.dev exporters append /v1/traces, /v1/logs, /v1/metrics. + GRAFANA_INSTANCE_ID: z.string().min(1).optional(), // Grafana Cloud instance ID (Basic auth username for OTLP) + GRAFANA_API_TOKEN: z.string().min(1).optional(), // Grafana Cloud API token (Basic auth password for OTLP) // External Services BROWSERBASE_API_KEY: z.string().min(1).optional(), // Browserbase API key for browser automation diff --git a/apps/sim/package.json b/apps/sim/package.json index ecce453c21e..bc6cea28920 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -65,6 +65,8 @@ "@modelcontextprotocol/sdk": "1.29.0", "@monaco-editor/react": "4.7.0", "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.217.0", + "@opentelemetry/exporter-metrics-otlp-proto": "^0.217.0", "@opentelemetry/exporter-trace-otlp-http": "^0.217.0", "@opentelemetry/resources": "^2.7.0", "@opentelemetry/sdk-node": "^0.217.0", diff --git a/apps/sim/trigger.config.ts b/apps/sim/trigger.config.ts index 41685d0010e..d19f3fb596f 100644 --- a/apps/sim/trigger.config.ts +++ b/apps/sim/trigger.config.ts @@ -1,7 +1,36 @@ +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http' +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto' +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http' import { additionalFiles, additionalPackages } from '@trigger.dev/build/extensions/core' import { defineConfig } from '@trigger.dev/sdk' import { env } from './lib/core/config/env' +const grafanaEndpoint = env.GRAFANA_OTLP_ENDPOINT +const grafanaInstanceId = env.GRAFANA_INSTANCE_ID +const grafanaToken = env.GRAFANA_API_TOKEN +const grafanaConfigured = Boolean(grafanaEndpoint || grafanaInstanceId || grafanaToken) +const grafanaFullyConfigured = Boolean(grafanaEndpoint && grafanaInstanceId && grafanaToken) + +if (grafanaConfigured && !grafanaFullyConfigured) { + throw new Error( + 'Grafana OTLP telemetry is partially configured. Set GRAFANA_OTLP_ENDPOINT, GRAFANA_INSTANCE_ID, and GRAFANA_API_TOKEN together, or leave all three unset.' + ) +} + +const grafanaTelemetry = grafanaFullyConfigured + ? (() => { + const baseUrl = grafanaEndpoint!.replace(/\/+$/, '') + const headers = { + Authorization: `Basic ${Buffer.from(`${grafanaInstanceId}:${grafanaToken}`).toString('base64')}`, + } + return { + exporters: [new OTLPTraceExporter({ url: `${baseUrl}/v1/traces`, headers })], + logExporters: [new OTLPLogExporter({ url: `${baseUrl}/v1/logs`, headers })], + metricExporters: [new OTLPMetricExporter({ url: `${baseUrl}/v1/metrics`, headers })], + } + })() + : undefined + export default defineConfig({ project: env.TRIGGER_PROJECT_ID!, runtime: 'node-22', @@ -14,6 +43,7 @@ export default defineConfig({ }, }, dirs: ['./background'], + ...(grafanaTelemetry ? { telemetry: grafanaTelemetry } : {}), build: { external: ['isolated-vm'], extensions: [ diff --git a/bun.lock b/bun.lock index 06d495404cd..f5963e861c2 100644 --- a/bun.lock +++ b/bun.lock @@ -120,6 +120,8 @@ "@modelcontextprotocol/sdk": "1.29.0", "@monaco-editor/react": "4.7.0", "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.217.0", + "@opentelemetry/exporter-metrics-otlp-proto": "^0.217.0", "@opentelemetry/exporter-trace-otlp-http": "^0.217.0", "@opentelemetry/resources": "^2.7.0", "@opentelemetry/sdk-node": "^0.217.0", From 222089a66a60ff09cc46c88b1a81c01754dbfb8a Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 13 May 2026 15:30:57 -0700 Subject: [PATCH 2/4] improvement(observability): use OTLP HTTP/JSON for metrics for consistency with traces and logs --- apps/sim/package.json | 2 +- apps/sim/trigger.config.ts | 2 +- bun.lock | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/sim/package.json b/apps/sim/package.json index bc6cea28920..1026ae72ba7 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -66,7 +66,7 @@ "@monaco-editor/react": "4.7.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-logs-otlp-http": "^0.217.0", - "@opentelemetry/exporter-metrics-otlp-proto": "^0.217.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.217.0", "@opentelemetry/exporter-trace-otlp-http": "^0.217.0", "@opentelemetry/resources": "^2.7.0", "@opentelemetry/sdk-node": "^0.217.0", diff --git a/apps/sim/trigger.config.ts b/apps/sim/trigger.config.ts index d19f3fb596f..999bf7012de 100644 --- a/apps/sim/trigger.config.ts +++ b/apps/sim/trigger.config.ts @@ -1,5 +1,5 @@ import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http' -import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-proto' +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http' import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http' import { additionalFiles, additionalPackages } from '@trigger.dev/build/extensions/core' import { defineConfig } from '@trigger.dev/sdk' diff --git a/bun.lock b/bun.lock index f5963e861c2..07afdcca315 100644 --- a/bun.lock +++ b/bun.lock @@ -121,7 +121,7 @@ "@monaco-editor/react": "4.7.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-logs-otlp-http": "^0.217.0", - "@opentelemetry/exporter-metrics-otlp-proto": "^0.217.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.217.0", "@opentelemetry/exporter-trace-otlp-http": "^0.217.0", "@opentelemetry/resources": "^2.7.0", "@opentelemetry/sdk-node": "^0.217.0", From a05317db293167a5da3fd65af0acc448e24cd912 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 13 May 2026 15:59:15 -0700 Subject: [PATCH 3/4] feat(observability): tag Trigger.dev telemetry with deployment.environment.name --- apps/sim/lib/core/config/env.ts | 1 + apps/sim/trigger.config.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 5b532af563f..48e573d5532 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -160,6 +160,7 @@ export const env = createEnv({ GRAFANA_OTLP_ENDPOINT: z.string().url().optional(), // Grafana Cloud OTLP HTTP gateway base URL (e.g., https://otlp-gateway-prod-us-east-0.grafana.net/otlp). Trigger.dev exporters append /v1/traces, /v1/logs, /v1/metrics. GRAFANA_INSTANCE_ID: z.string().min(1).optional(), // Grafana Cloud instance ID (Basic auth username for OTLP) GRAFANA_API_TOKEN: z.string().min(1).optional(), // Grafana Cloud API token (Basic auth password for OTLP) + SIM_DEPLOYMENT_ENVIRONMENT: z.string().min(1).optional(), // Deployment tier label (e.g., "production", "staging", "development"). Emitted as the stable `deployment.environment.name` resource attribute on Trigger.dev telemetry to match the rest of the Sim OTEL stack. // External Services BROWSERBASE_API_KEY: z.string().min(1).optional(), // Browserbase API key for browser automation diff --git a/apps/sim/trigger.config.ts b/apps/sim/trigger.config.ts index 999bf7012de..c1fb526a36c 100644 --- a/apps/sim/trigger.config.ts +++ b/apps/sim/trigger.config.ts @@ -1,6 +1,7 @@ import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http' import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http' import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http' +import { resourceFromAttributes } from '@opentelemetry/resources' import { additionalFiles, additionalPackages } from '@trigger.dev/build/extensions/core' import { defineConfig } from '@trigger.dev/sdk' import { env } from './lib/core/config/env' @@ -23,10 +24,15 @@ const grafanaTelemetry = grafanaFullyConfigured const headers = { Authorization: `Basic ${Buffer.from(`${grafanaInstanceId}:${grafanaToken}`).toString('base64')}`, } + const deploymentEnvironment = env.SIM_DEPLOYMENT_ENVIRONMENT + const resource = deploymentEnvironment + ? resourceFromAttributes({ 'deployment.environment.name': deploymentEnvironment }) + : undefined return { exporters: [new OTLPTraceExporter({ url: `${baseUrl}/v1/traces`, headers })], logExporters: [new OTLPLogExporter({ url: `${baseUrl}/v1/logs`, headers })], metricExporters: [new OTLPMetricExporter({ url: `${baseUrl}/v1/metrics`, headers })], + ...(resource ? { resource } : {}), } })() : undefined From 2892b4fdc322fb441d843e3e97c94d6e8bb47f86 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Wed, 13 May 2026 16:05:12 -0700 Subject: [PATCH 4/4] improvement(observability): switch Grafana telemetry vars to OTLP-shaped trio --- apps/sim/lib/core/config/env.ts | 5 ++-- apps/sim/trigger.config.ts | 46 ++++++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 48e573d5532..26d4cd4ad42 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -158,9 +158,8 @@ export const env = createEnv({ PROFOUND_API_KEY: z.string().min(1).optional(), // Profound analytics API key PROFOUND_ENDPOINT: z.string().url().optional(), // Profound analytics endpoint GRAFANA_OTLP_ENDPOINT: z.string().url().optional(), // Grafana Cloud OTLP HTTP gateway base URL (e.g., https://otlp-gateway-prod-us-east-0.grafana.net/otlp). Trigger.dev exporters append /v1/traces, /v1/logs, /v1/metrics. - GRAFANA_INSTANCE_ID: z.string().min(1).optional(), // Grafana Cloud instance ID (Basic auth username for OTLP) - GRAFANA_API_TOKEN: z.string().min(1).optional(), // Grafana Cloud API token (Basic auth password for OTLP) - SIM_DEPLOYMENT_ENVIRONMENT: z.string().min(1).optional(), // Deployment tier label (e.g., "production", "staging", "development"). Emitted as the stable `deployment.environment.name` resource attribute on Trigger.dev telemetry to match the rest of the Sim OTEL stack. + GRAFANA_OTLP_HEADERS: z.string().min(1).optional(), // Comma-separated key=value headers for OTLP requests (e.g., "Authorization=Basic "). Same format as the OTEL_EXPORTER_OTLP_HEADERS spec. + GRAFANA_DEPLOYMENT_ENVIRONMENT: z.string().min(1).optional(), // Deployment tier label (e.g., "production", "staging", "development"). Emitted as the stable `deployment.environment.name` resource attribute on Trigger.dev telemetry to match the rest of the Sim OTEL stack. // External Services BROWSERBASE_API_KEY: z.string().min(1).optional(), // Browserbase API key for browser automation diff --git a/apps/sim/trigger.config.ts b/apps/sim/trigger.config.ts index c1fb526a36c..1439f5af2fd 100644 --- a/apps/sim/trigger.config.ts +++ b/apps/sim/trigger.config.ts @@ -7,32 +7,54 @@ import { defineConfig } from '@trigger.dev/sdk' import { env } from './lib/core/config/env' const grafanaEndpoint = env.GRAFANA_OTLP_ENDPOINT -const grafanaInstanceId = env.GRAFANA_INSTANCE_ID -const grafanaToken = env.GRAFANA_API_TOKEN -const grafanaConfigured = Boolean(grafanaEndpoint || grafanaInstanceId || grafanaToken) -const grafanaFullyConfigured = Boolean(grafanaEndpoint && grafanaInstanceId && grafanaToken) +const grafanaHeaders = env.GRAFANA_OTLP_HEADERS +const grafanaDeploymentEnvironment = env.GRAFANA_DEPLOYMENT_ENVIRONMENT +const grafanaConfigured = Boolean(grafanaEndpoint || grafanaHeaders || grafanaDeploymentEnvironment) +const grafanaFullyConfigured = Boolean( + grafanaEndpoint && grafanaHeaders && grafanaDeploymentEnvironment +) if (grafanaConfigured && !grafanaFullyConfigured) { throw new Error( - 'Grafana OTLP telemetry is partially configured. Set GRAFANA_OTLP_ENDPOINT, GRAFANA_INSTANCE_ID, and GRAFANA_API_TOKEN together, or leave all three unset.' + 'Grafana OTLP telemetry is partially configured. Set GRAFANA_OTLP_ENDPOINT, GRAFANA_OTLP_HEADERS, and GRAFANA_DEPLOYMENT_ENVIRONMENT together, or leave all three unset.' ) } +/** + * Parse OTLP headers per the OTEL spec format `key1=value1,key2=value2`. + * Values are URL-decoded; keys/values are trimmed; empty entries are skipped. + * @see https://opentelemetry.io/docs/specs/otel/protocol/exporter/ + */ +function parseOtlpHeaders(raw: string): Record { + const out: Record = {} + for (const pair of raw.split(',')) { + const eq = pair.indexOf('=') + if (eq === -1) continue + const key = pair.slice(0, eq).trim() + const value = pair.slice(eq + 1).trim() + if (!key) continue + out[key] = decodeURIComponent(value) + } + return out +} + const grafanaTelemetry = grafanaFullyConfigured ? (() => { const baseUrl = grafanaEndpoint!.replace(/\/+$/, '') - const headers = { - Authorization: `Basic ${Buffer.from(`${grafanaInstanceId}:${grafanaToken}`).toString('base64')}`, + const headers = parseOtlpHeaders(grafanaHeaders!) + if (Object.keys(headers).length === 0) { + throw new Error( + 'GRAFANA_OTLP_HEADERS is set but yielded no valid key=value pairs. Expected format: "key1=value1,key2=value2".' + ) } - const deploymentEnvironment = env.SIM_DEPLOYMENT_ENVIRONMENT - const resource = deploymentEnvironment - ? resourceFromAttributes({ 'deployment.environment.name': deploymentEnvironment }) - : undefined + const resource = resourceFromAttributes({ + 'deployment.environment.name': grafanaDeploymentEnvironment!, + }) return { exporters: [new OTLPTraceExporter({ url: `${baseUrl}/v1/traces`, headers })], logExporters: [new OTLPLogExporter({ url: `${baseUrl}/v1/logs`, headers })], metricExporters: [new OTLPMetricExporter({ url: `${baseUrl}/v1/metrics`, headers })], - ...(resource ? { resource } : {}), + resource, } })() : undefined