fix(webapp): use provider-reported cost for AI generations when present (#4186)

## Summary

The run page could show an AI generation cost well above what the
provider actually charged, most visibly for OpenRouter and Vercel AI
Gateway requests where a heavily cache-read prompt was priced at the
full input rate. When the provider reports an exact per-request cost, we
now use that instead of catalog pricing.

## Fix

Gateway and OpenRouter include the exact per-request cost in
`ai.response.providerMetadata` (`openrouter.usage.cost` /
`gateway.cost`). That figure already reflects the cache-read discount
and the real per-provider rate, which the catalog cannot reconstruct:
cache-read counts do not arrive in `gen_ai.usage.*`, and per-model
catalog prices drift from what the provider billed, in either direction.
So provider-reported cost is now preferred, and the catalog is used only
when no provider cost is present.

Fallback routing is covered by the same change: when OpenRouter routes
to a different model, `gen_ai.response.model` already carries the served
model, so the cost follows the served model and the provider's own
figure makes it exact.

`extractProviderCost` now runs on every AI span, so it gets a cheap
`"cost"` substring guard to skip the JSON parse on reasoning-model spans
whose provider metadata carries large reasoning text and no cost field.

Regression tests cover the cache-discount overcharge, fallback
served-model pricing, gateway cost, and the catalog fallback path.

Author: ericallam

.server-changes/fix-llm-cost-prefer-provider-reported.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: fix
+---
+
+AI generation cost now uses the exact provider-reported cost from OpenRouter/Vercel AI Gateway when present, instead of catalog pricing, so cache-discounted and fallback-routed requests match the amount the provider actually billed.

apps/webapp/app/v3/utils/enrichCreatableEvents.server.test.ts

@@ -0,0 +1,214 @@
+import { describe, it, expect } from "vitest";
+import { enrichCreatableEvents, setLlmPricingRegistry } from "./enrichCreatableEvents.server";
+import type { CreateEventInput } from "../eventRepository/eventRepository.types";
+
+type Registry = Parameters<typeof setLlmPricingRegistry>[0];
+type RegistryCost = NonNullable<ReturnType<Registry["calculateCost"]>>;
+
+function registryReturning(cost: RegistryCost | null, isLoaded = true): Registry {
+  return {
+    isLoaded,
+    calculateCost: () => cost,
+  };
+}
+
+// A catalog cost result the registry would produce. The numbers here stand in for a catalog
+// price that disagrees with what the provider actually billed.
+function catalogCost(overrides: Partial<RegistryCost> = {}): RegistryCost {
+  return {
+    matchedModelId: "llm_model_test",
+    matchedModelName: "test-model",
+    pricingTierId: "tier_standard",
+    pricingTierName: "Standard",
+    inputCost: 0,
+    outputCost: 0,
+    totalCost: 0,
+    costDetails: {},
+    ...overrides,
+  };
+}
+
+function makeEvent(properties: Record<string, unknown>): CreateEventInput {
+  return {
+    message: "ai.generateText.doGenerate",
+    kind: "INTERNAL",
+    isPartial: false,
+    properties: properties as CreateEventInput["properties"],
+  } as unknown as CreateEventInput;
+}
+
+function enrichOne(event: CreateEventInput): CreateEventInput {
+  const [out] = enrichCreatableEvents([event]);
+  return out;
+}
+
+function costPillText(event: CreateEventInput): string | undefined {
+  const accessory = (event.style as any)?.accessory;
+  const items: Array<{ text: string; icon: string }> = accessory?.items ?? [];
+  return items.find((i) => i.icon === "tabler-currency-dollar")?.text;
+}
+
+function modelPillText(event: CreateEventInput): string | undefined {
+  const accessory = (event.style as any)?.accessory;
+  const items: Array<{ text: string; icon: string }> = accessory?.items ?? [];
+  return items.find((i) => i.icon === "tabler-cube")?.text;
+}
+
+describe("enrichLlmMetrics — provider-reported cost", () => {
+  it("prefers the provider-reported cost over catalog pricing when a cache discount applies", () => {
+    // OpenRouter served mimo with 25,280 of 34,374 prompt tokens as cache reads. Cache counts
+    // only reach us via providerMetadata, so the catalog bills the full prompt at the input
+    // rate while OpenRouter's exact bill reflects the cache discount.
+    setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.01603584 })));
+
+    const out = enrichOne(
+      makeEvent({
+        "gen_ai.system": "openrouter",
+        "gen_ai.request.model": "xiaomi/mimo-v2.5-pro",
+        "gen_ai.response.model": "xiaomi/mimo-v2.5-pro-20260422",
+        "gen_ai.usage.input_tokens": 34374,
+        "gen_ai.usage.output_tokens": 1245,
+        "ai.response.providerMetadata": JSON.stringify({
+          openrouter: {
+            provider: "Xiaomi",
+            usage: { cost: 0.005130048, promptTokensDetails: { cachedTokens: 25280 } },
+          },
+        }),
+      })
+    );
+
+    expect(out.properties["trigger.llm.total_cost"]).toBe(0.005130048);
+    expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter");
+    // The catalog breakdown must not be written — it priced the wrong (full-rate) total.
+    expect(out.properties["trigger.llm.input_cost"]).toBeUndefined();
+    expect(out.properties["trigger.llm.matched_model"]).toBeUndefined();
+    expect(costPillText(out)).toBe("$0.005130");
+
+    expect(out._llmMetrics?.totalCost).toBe(0.005130048);
+    expect(out._llmMetrics?.costSource).toBe("openrouter");
+    expect(out._llmMetrics?.providerCost).toBe(0.005130048);
+    expect(out._llmMetrics?.inputCost).toBe(0);
+  });
+
+  it("prices the served fallback model's provider cost, not the requested model", () => {
+    // Requested mimo, OpenRouter routed to a gemini fallback: gen_ai.response.model carries the
+    // SERVED model, and the provider cost is authoritative.
+    setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.02 })));
+
+    const out = enrichOne(
+      makeEvent({
+        "gen_ai.system": "openrouter",
+        "gen_ai.request.model": "xiaomi/mimo-v2.5-pro",
+        "gen_ai.response.model": "google/gemini-3.5-flash-20260519",
+        "gen_ai.usage.input_tokens": 5000,
+        "gen_ai.usage.output_tokens": 800,
+        "ai.response.providerMetadata": JSON.stringify({
+          openrouter: { provider: "Google", usage: { cost: 0.011058 } },
+        }),
+      })
+    );
+
+    expect(out.properties["trigger.llm.total_cost"]).toBe(0.011058);
+    expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter");
+    // The pill (and stored response model) reflect the served fallback model.
+    expect(modelPillText(out)).toBe("google/gemini-3.5-flash-20260519");
+    expect(out._llmMetrics?.requestModel).toBe("xiaomi/mimo-v2.5-pro");
+    expect(out._llmMetrics?.responseModel).toBe("google/gemini-3.5-flash-20260519");
+    expect(out._llmMetrics?.totalCost).toBe(0.011058);
+  });
+
+  it("prefers the gateway-reported cost over catalog pricing", () => {
+    setLlmPricingRegistry(registryReturning(catalogCost({ totalCost: 0.02 })));
+
+    const out = enrichOne(
+      makeEvent({
+        "gen_ai.system": "gateway",
+        "gen_ai.response.model": "openai/gpt-4o",
+        "gen_ai.usage.input_tokens": 1000,
+        "gen_ai.usage.output_tokens": 500,
+        "ai.response.providerMetadata": JSON.stringify({
+          gateway: { cost: "0.0006615" },
+        }),
+      })
+    );
+
+    expect(out.properties["trigger.llm.total_cost"]).toBe(0.0006615);
+    expect(out.properties["trigger.llm.cost_source"]).toBe("gateway");
+    expect(out._llmMetrics?.costSource).toBe("gateway");
+  });
+
+  it("uses provider cost even when the catalog does not match the model", () => {
+    setLlmPricingRegistry(registryReturning(null));
+
+    const out = enrichOne(
+      makeEvent({
+        "gen_ai.system": "openrouter",
+        "gen_ai.response.model": "some/unlisted-model",
+        "gen_ai.usage.input_tokens": 2000,
+        "gen_ai.usage.output_tokens": 300,
+        "ai.response.providerMetadata": JSON.stringify({
+          openrouter: { provider: "SomeProvider", usage: { cost: 0.00042 } },
+        }),
+      })
+    );
+
+    expect(out.properties["trigger.llm.total_cost"]).toBe(0.00042);
+    expect(out.properties["trigger.llm.cost_source"]).toBe("openrouter");
+  });
+
+  it("falls back to catalog pricing when no provider cost is reported", () => {
+    setLlmPricingRegistry(
+      registryReturning(
+        catalogCost({
+          matchedModelId: "llm_model_gpt4o",
+          matchedModelName: "gpt-4o",
+          inputCost: 0.04,
+          outputCost: 0.01,
+          totalCost: 0.05,
+          costDetails: { input: 0.04, output: 0.01 },
+        })
+      )
+    );
+
+    const out = enrichOne(
+      makeEvent({
+        "gen_ai.system": "openai",
+        "gen_ai.response.model": "gpt-4o",
+        "gen_ai.usage.input_tokens": 1000,
+        "gen_ai.usage.output_tokens": 500,
+      })
+    );
+
+    expect(out.properties["trigger.llm.total_cost"]).toBe(0.05);
+    expect(out.properties["trigger.llm.input_cost"]).toBe(0.04);
+    expect(out.properties["trigger.llm.output_cost"]).toBe(0.01);
+    expect(out.properties["trigger.llm.matched_model"]).toBe("gpt-4o");
+    // Registry path does not set a cost_source attribute.
+    expect(out.properties["trigger.llm.cost_source"]).toBeUndefined();
+    expect(out._llmMetrics?.costSource).toBe("registry");
+    expect(out._llmMetrics?.matchedModelId).toBe("llm_model_gpt4o");
+  });
+
+  it("falls back to catalog pricing when providerMetadata carries no cost field", () => {
+    // The cheap `"cost"` guard should skip parsing and let the registry price this span.
+    setLlmPricingRegistry(
+      registryReturning(catalogCost({ totalCost: 0.03, matchedModelName: "claude" }))
+    );
+
+    const out = enrichOne(
+      makeEvent({
+        "gen_ai.system": "anthropic",
+        "gen_ai.response.model": "claude-sonnet-4-0",
+        "gen_ai.usage.input_tokens": 1000,
+        "gen_ai.usage.output_tokens": 500,
+        "ai.response.providerMetadata": JSON.stringify({
+          anthropic: { usage: { service_tier: "standard" } },
+        }),
+      })
+    );
+
+    expect(out.properties["trigger.llm.total_cost"]).toBe(0.03);
+    expect(out.properties["trigger.llm.cost_source"]).toBeUndefined();
+    expect(out._llmMetrics?.costSource).toBe("registry");
+  });
+});

apps/webapp/app/v3/utils/enrichCreatableEvents.server.ts

@@ -89,20 +89,17 @@ function enrichLlmMetrics(event: CreateEventInput): void {
     { text: formatTokenCount(totalTokens), icon: "tabler-hash" },
   ];
 
-  // Try cost enrichment if the registry is loaded.
-  // The registry handles prefix stripping (e.g. "mistral/mistral-large-3" → "mistral-large-3")
-  // for gateway/openrouter models automatically in its match() method.
+  // Provider-reported cost (gateway/openrouter) is the exact per-request bill and already
+  // reflects cache-read discounts and the real per-provider rate, so prefer it and only fall
+  // back to catalog pricing when it is absent. The registry handles prefix stripping (e.g.
+  // "mistral/mistral-large-3" → "mistral-large-3") for gateway/openrouter models in match().
+  const providerCost = extractProviderCost(props);
+
   let cost: ReturnType<NonNullable<typeof _registry>["calculateCost"]> | null = null;
-  if (_registry?.isLoaded) {
+  if (!providerCost && _registry?.isLoaded) {
     cost = _registry.calculateCost(responseModel, usageDetails);
   }
 
-  // Fallback: extract cost from provider metadata (gateway/openrouter report per-request cost)
-  let providerCost: { totalCost: number; source: string } | null = null;
-  if (!cost) {
-    providerCost = extractProviderCost(props);
-  }
-
   if (cost) {
     // Add trigger.llm.* attributes to the span from our pricing registry
     event.properties = {
@@ -342,6 +339,11 @@ function extractProviderCost(
   const rawMeta = props["ai.response.providerMetadata"];
   if (typeof rawMeta !== "string") return null;
 
+  // Cheap guard: providerMetadata can be large for reasoning models (it carries the full
+  // reasoning_details text), and this now runs on every AI span. Skip the JSON parse when
+  // there is no cost field to find.
+  if (!rawMeta.includes('"cost"')) return null;
+
   let meta: Record<string, unknown>;
   try {
     meta = JSON.parse(rawMeta) as Record<string, unknown>;

apps/webapp/vitest.config.ts

@@ -13,6 +13,7 @@ export default defineConfig({
       "test/**/*.test.ts",
       "app/v3/runOpsMigration/**/*.test.ts",
       "app/v3/runStore.server.test.ts",
+      "app/v3/utils/**/*.test.ts",
       "app/v3/services/bulk/**/*.test.ts",
       "app/runEngine/concerns/**/*.test.ts",
       "app/runEngine/services/**/*.test.ts",