diff --git a/apps/landing/src/content/docs/sdks/effect-cloudflare.md b/apps/landing/src/content/docs/sdks/effect-cloudflare.md index afc0c537d..c1b99c1df 100644 --- a/apps/landing/src/content/docs/sdks/effect-cloudflare.md +++ b/apps/landing/src/content/docs/sdks/effect-cloudflare.md @@ -66,6 +66,8 @@ In addition to the [common options](/docs/sdks/effect#configuration-reference), `anticipatedErrorIdentifiers` keeps expected rejections (a 404, a 401) visible as traces without counting them as errors — matching how Maple's ingest gateway treats 4xx. A span still exports as `Error` if its cause contains any defect. +An error that crossed an HTTP boundary is a decoded body rather than the class that raised it, so a failure shaped `{ error: { _tag } }` — the envelope convention many APIs use — is matched on the body's `_tag`. Client-side spans classify the same as the server-side ones they mirror, with no separate identifiers to configure. + `dropSpanNames` is useful for suppressing protocol-level chatter — e.g. `["McpServer/Notifications."]` to drop MCP notification spam without dropping legitimate handler spans. ## Endpoint Resolution diff --git a/apps/web/src/api/warehouse/effect-utils.test.ts b/apps/web/src/api/warehouse/effect-utils.test.ts index fbdda8e61..c786d6ed8 100644 --- a/apps/web/src/api/warehouse/effect-utils.test.ts +++ b/apps/web/src/api/warehouse/effect-utils.test.ts @@ -1,6 +1,19 @@ -import { describe, expect, it } from "vitest" +import { beforeEach, describe, expect, it } from "vitest" import { WarehouseQuotaExceededError } from "@maple/domain/http" -import { WarehouseDecodeError, WarehouseQueryError, normalizeWarehouseError } from "./effect-utils" +import { HttpClientError, HttpClientRequest } from "effect/unstable/http" +import { apiBaseUrl } from "@/lib/services/common/api-base-url" +import { + noteReachable, + noteUnreachable, + originOf, + PEER_OUTAGE_GRACE_MS, +} from "@/lib/services/common/peer-reachability" +import { + WarehouseDecodeError, + WarehouseQueryError, + WarehouseUnreachableError, + normalizeWarehouseError, +} from "./effect-utils" describe("normalizeWarehouseError", () => { it("preserves a v2 error envelope", () => { @@ -37,3 +50,58 @@ describe("normalizeWarehouseError", () => { expect(normalizeWarehouseError("query", normalized)).toBe(normalized) }) }) + +const transportFailure = () => + new HttpClientError.HttpClientError({ + reason: new HttpClientError.TransportError({ + request: HttpClientRequest.post(`${apiBaseUrl}/internal/query-engine/execute-batch`), + cause: new TypeError("Failed to fetch"), + }), + }) + +/** + * A dropped connection is the network, not the warehouse — but only while it is + * still short enough to be a blip. Past the grace window the API is genuinely + * unreachable and the failure reports exactly as it did before. + */ +describe("transport failures during a connectivity blip", () => { + const origin = originOf(apiBaseUrl) + + beforeEach(() => { + noteReachable(origin) + }) + + it("classifies a transport failure as unreachable while the origin is blipping", () => { + noteUnreachable(origin, Date.now()) + + const normalized = normalizeWarehouseError("query", transportFailure()) + + expect(normalized).toBeInstanceOf(WarehouseUnreachableError) + if (!(normalized instanceof WarehouseUnreachableError)) throw new Error("expected unreachable") + // The user is told the API is unreachable and that it retries — never that + // their query needs fixing. + expect(normalized.error.title).toBe("Cannot reach Maple API") + expect(normalized.error.retryable).toBe(true) + }) + + it("unwraps a transport failure nested behind another error", () => { + noteUnreachable(origin, Date.now()) + const wrapped = new Error("Warehouse batch request failed", { cause: transportFailure() }) + + expect(normalizeWarehouseError("query", wrapped)).toBeInstanceOf(WarehouseUnreachableError) + }) + + it("reports a transport failure once the outage outlasts the grace window", () => { + noteUnreachable(origin, Date.now() - PEER_OUTAGE_GRACE_MS - 1) + + expect(normalizeWarehouseError("query", transportFailure())).toBeInstanceOf(WarehouseQueryError) + }) + + it("leaves a failure the API actually answered alone, blip or not", () => { + noteUnreachable(origin, Date.now()) + + expect(normalizeWarehouseError("query", new Error("decode failed"))).toBeInstanceOf( + WarehouseQueryError, + ) + }) +}) diff --git a/apps/web/src/api/warehouse/effect-utils.ts b/apps/web/src/api/warehouse/effect-utils.ts index ec455994b..5de5aedbe 100644 --- a/apps/web/src/api/warehouse/effect-utils.ts +++ b/apps/web/src/api/warehouse/effect-utils.ts @@ -8,6 +8,7 @@ import { type AttributeValueItem, } from "@maple/query-engine" import { Effect, Layer, Schema } from "effect" +import { HttpClientError } from "effect/unstable/http" import { PublicHttpErrorBodySchema, type AnyPublicHttpErrorBody } from "@maple/domain/http" import { MapleApiAtomClient } from "@/lib/services/common/atom-client" import { MapleInternalAtomClient } from "@/lib/services/common/internal-atom-client" @@ -18,7 +19,9 @@ import { mapleInternalClientLayer, mapleRuntime, } from "@/lib/registry" -import { makeClientErrorBody } from "@/lib/error-messages" +import { makeClientErrorBody, NetworkErrorBody } from "@/lib/error-messages" +import { apiBaseUrl } from "@/lib/services/common/api-base-url" +import { isBlipping, originOf } from "@/lib/services/common/peer-reachability" import { makeExecuteBatcher } from "./execute-batcher" export const WarehouseDateTimeString = TinybirdDateTime @@ -94,11 +97,39 @@ export class WarehouseInvalidInputError extends Schema.TaggedError()( + "@maple/web/errors/WarehouseUnreachableError", + { + operation: Schema.String, + message: Schema.String, + cause: Schema.optional(Schema.Unknown), + }, +) { + readonly error = NetworkErrorBody +} + export type WarehouseApiError = | WarehouseDecodeError | WarehouseQueryError | WarehouseTransformError | WarehouseInvalidInputError + | WarehouseUnreachableError /** Backend failures are either a public body or an error carrying that same body. */ export type BackendError = AnyPublicHttpErrorBody | { readonly error: AnyPublicHttpErrorBody } @@ -124,17 +155,43 @@ export const isWarehouseApiError = (cause: unknown): cause is WarehouseApiError typeof cause._tag === "string" && cause._tag.startsWith("@maple/web/errors/Warehouse") +/** + * True when `cause` is a request that never got a response — the browser could + * not reach the API — as opposed to one the API answered with a failure. + * + * Walks the cause chain because the transport failure is usually nested: the + * batcher rejects its promise with an `HttpClientError`, which `Effect.tryPromise` + * then wraps. Bounded at the same depth `displayError` uses. + */ +export const isTransportFailure = (cause: unknown, depth = 0): boolean => { + if (HttpClientError.isHttpClientError(cause)) return cause.reason._tag === "TransportError" + if (depth >= 4) return false + const nested = + typeof cause === "object" && cause !== null && "cause" in cause + ? (cause as { readonly cause: unknown }).cause + : undefined + return nested === undefined || nested === cause ? false : isTransportFailure(nested, depth + 1) +} + +/** + * A transport failure while the API is inside its grace window is the network + * dropping, not the warehouse failing — see `peer-reachability.ts`. Once the run + * outlasts the window it is a real outage and stays a `WarehouseQueryError`, so + * an API that is genuinely down still reports. + */ +export const isNetworkBlip = (cause: unknown): boolean => + isTransportFailure(cause) && isBlipping(originOf(apiBaseUrl), Date.now()) + /** Preserve known errors; introduce a local query error only for an unstructured failure. */ export const normalizeWarehouseError = ( operation: string, cause: unknown, ): WarehouseApiError | BackendError => { if (isBackendError(cause) || isWarehouseApiError(cause)) return cause - return new WarehouseQueryError({ - operation, - message: toMessage(cause, `Warehouse query failed for ${operation}`), - cause, - }) + const message = toMessage(cause, `Warehouse query failed for ${operation}`) + return isNetworkBlip(cause) + ? new WarehouseUnreachableError({ operation, message, cause }) + : new WarehouseQueryError({ operation, message, cause }) } export function decodeInput( @@ -195,6 +252,25 @@ export function runWarehouseQueryV2( ) } +/** + * Raise a query-set failure whose per-query causes the runner has already + * flattened to strings. + * + * `runQuerySetWindow` catches each executor failure into `result.error` text and + * re-raises the batch as `QuerySetNoDataError`, so the type is gone by the time + * an adapter sees it and only the live reachability clock still knows whether + * the API answered. While it says the API is unreachable this is that, not a bad + * query — which is what the user was previously told to fix. + */ +export function querySetFailure( + operation: string, + message: string, +): Effect.Effect { + return isBlipping(originOf(apiBaseUrl), Date.now()) + ? Effect.fail(new WarehouseUnreachableError({ operation, message })) + : invalidWarehouseInput(operation, message) +} + export function invalidWarehouseInput( operation: string, message: string, @@ -229,12 +305,7 @@ const executeQueryEngineEffect = Effect.fn("QueryEngine.execute")(function* ( ) { return yield* Effect.tryPromise({ try: () => executeBatcher.enqueue(payload), - catch: (cause) => - new WarehouseQueryError({ - operation: "QueryEngine.executeBatch", - message: toMessage(cause, "Warehouse batch request failed"), - cause, - }), + catch: (cause) => normalizeWarehouseError("QueryEngine.executeBatch", cause), }) }) diff --git a/apps/web/src/api/warehouse/query-builder-breakdown.test.ts b/apps/web/src/api/warehouse/query-builder-breakdown.test.ts index 6f062fe86..dbca76ce7 100644 --- a/apps/web/src/api/warehouse/query-builder-breakdown.test.ts +++ b/apps/web/src/api/warehouse/query-builder-breakdown.test.ts @@ -1,5 +1,7 @@ +import { Effect, Exit } from "effect" import { describe, expect, it } from "vitest" +import { QuerySetNoDataError } from "@maple/query-engine/query-set" import * as breakdownModule from "@/api/warehouse/query-builder-breakdown" // `mergeBreakdownResults` moved to `@maple/query-engine/query-set`, where the @@ -13,3 +15,37 @@ describe("query-builder breakdown units", () => { expect(breakdownModule.__testables).not.toHaveProperty("normalizeErrorRatePoints") }) }) + +/** + * The timeseries adapter stopped failing on an empty window — an empty window is + * a normal answer — but the breakdown adapter beside it was left raising + * `WarehouseInvalidInputError`, which marked the span `Error` and billed an + * exception event for a panel the user simply has no data for. + */ +describe("empty window", () => { + it("answers with zero rows when every query ran and none matched", () => { + const exit = Effect.runSyncExit( + breakdownModule.__testables.onNoData( + new QuerySetNoDataError({ + message: "No breakdown data found in selected time range", + details: [], + }), + ), + ) + + expect(exit).toStrictEqual(Exit.succeed({ rows: [], diagnostics: [] })) + }) + + it("still fails when a query itself failed", () => { + const exit = Effect.runSyncExit( + breakdownModule.__testables.onNoData( + new QuerySetNoDataError({ + message: "Unknown column 'nope'", + details: ["Unknown column 'nope'"], + }), + ), + ) + + expect(Exit.isFailure(exit)).toBe(true) + }) +}) diff --git a/apps/web/src/api/warehouse/query-builder-breakdown.ts b/apps/web/src/api/warehouse/query-builder-breakdown.ts index cf52af09e..7fba30a02 100644 --- a/apps/web/src/api/warehouse/query-builder-breakdown.ts +++ b/apps/web/src/api/warehouse/query-builder-breakdown.ts @@ -1,7 +1,17 @@ import { Effect, Schema } from "effect" import { QueryBuilderQueryDraftSchema } from "@maple/domain/http" -import { runBreakdownQuerySet } from "@maple/query-engine/query-set" -import { decodeInput, invalidWarehouseInput } from "@/api/warehouse/effect-utils" +import { + runBreakdownQuerySet, + type BreakdownQuerySetResult, + type QuerySetNoDataError, +} from "@maple/query-engine/query-set" +import { + decodeInput, + invalidWarehouseInput, + querySetFailure, + type WarehouseInvalidInputError, + type WarehouseUnreachableError, +} from "@/api/warehouse/effect-utils" import { makeWarehouseExecutor } from "@/api/warehouse/query-set-executor" const executor = makeWarehouseExecutor("queryEngine.breakdownQuery") @@ -23,6 +33,22 @@ const QueryBuilderBreakdownInputSchema = Schema.Struct({ export type QueryBuilderBreakdownInput = Schema.Schema.Type +/** + * An empty window is a normal answer, not a failure. + * + * The same reasoning as `getQueryBuilderTimeseries`, which this path was left + * out of: failing here marked the span `Error` and billed an exception event for + * a panel the user simply has no data for, and `use-widget-data` already renders + * an empty envelope as the muted "No data" frame. A populated `details` carries + * a real per-query failure and stays an error. + */ +const onNoData = ( + error: QuerySetNoDataError, +): Effect.Effect => + error.details.length === 0 + ? Effect.succeed({ rows: [], diagnostics: [] }) + : querySetFailure("getQueryBuilderBreakdown", error.message) + export function getQueryBuilderBreakdown({ data }: { data: QueryBuilderBreakdownInput }) { return getQueryBuilderBreakdownEffect({ data }) } @@ -43,8 +69,7 @@ const getQueryBuilderBreakdownEffect = Effect.fn("QueryEngine.getQueryBuilderBre Effect.catchTags({ "@maple/query-engine/query-set/QuerySetInputError": (error) => invalidWarehouseInput("getQueryBuilderBreakdown", error.message), - "@maple/query-engine/query-set/QuerySetNoDataError": (error) => - invalidWarehouseInput("getQueryBuilderBreakdown", error.message), + "@maple/query-engine/query-set/QuerySetNoDataError": onNoData, }), ) @@ -54,4 +79,4 @@ const getQueryBuilderBreakdownEffect = Effect.fn("QueryEngine.getQueryBuilderBre // The merge and the per-query execution moved to `@maple/query-engine/query-set` // and are tested there; what stays worth asserting here is that this module adds // no rescaling of its own on the way out. -export const __testables = {} +export const __testables = { onNoData } diff --git a/apps/web/src/api/warehouse/query-builder-timeseries.ts b/apps/web/src/api/warehouse/query-builder-timeseries.ts index 1a7b8b3b1..6fbdedd87 100644 --- a/apps/web/src/api/warehouse/query-builder-timeseries.ts +++ b/apps/web/src/api/warehouse/query-builder-timeseries.ts @@ -9,7 +9,7 @@ import { fallbackStrategyFromWire, runTimeseriesQuerySet, } from "@maple/query-engine/query-set" -import { decodeInput, invalidWarehouseInput } from "@/api/warehouse/effect-utils" +import { decodeInput, invalidWarehouseInput, querySetFailure } from "@/api/warehouse/effect-utils" import { makeWarehouseExecutor } from "@/api/warehouse/query-set-executor" /** @@ -130,8 +130,12 @@ const getQueryBuilderTimeseriesEffect = Effect.fn("QueryEngine.getQueryBuilderTi ...(!(input.maxDataPoints === undefined) ? { maxDataPoints: input.maxDataPoints } : undefined), }).pipe( // The runner's tagged failures carry the message this app already showed; - // re-raising them as `WarehouseInvalidInputError` keeps `displayError` and - // `mapBuilderChartFailure` working unchanged. + // re-raising them keeps `displayError` and `mapBuilderChartFailure` working + // unchanged. `querySetFailure` rather than `invalidWarehouseInput` for the + // no-data case: the runner stringifies each per-query failure into + // `details`, so a dropped connection arrived here as text and was re-raised + // as "Invalid query" — telling the user to fix a request that never left + // the browser. Effect.catchTags({ "@maple/query-engine/query-set/QuerySetInputError": (error) => invalidWarehouseInput("getQueryBuilderTimeseries", error.message), @@ -144,7 +148,7 @@ const getQueryBuilderTimeseriesEffect = Effect.fn("QueryEngine.getQueryBuilderTi rows: [], diagnostics: emptyTimeseriesResponse(input).diagnostics, } satisfies TimeseriesQuerySetResult) - : invalidWarehouseInput("getQueryBuilderTimeseries", error.message), + : querySetFailure("getQueryBuilderTimeseries", error.message), }), ) diff --git a/apps/web/src/lib/error-messages.ts b/apps/web/src/lib/error-messages.ts index 6c0465ef4..1a2c8eb89 100644 --- a/apps/web/src/lib/error-messages.ts +++ b/apps/web/src/lib/error-messages.ts @@ -25,7 +25,12 @@ export const makeClientErrorBody = (definition: ClientErrorDefinition): AnyPubli ...definition, }) -const NetworkError = makeClientErrorBody({ +/** + * Shared so the warehouse layer can raise this same body as a typed failure — + * `WarehouseUnreachableError` — rather than only reaching it by unwrapping a + * cause chain. One copy, one tag, whichever way a dropped connection arrives. + */ +export const NetworkErrorBody = makeClientErrorBody({ _tag: NetworkErrorTag, code: "network_unreachable", title: "Cannot reach Maple API", @@ -111,7 +116,7 @@ const displayErrorInternal = (input: unknown, depth: number): AnyPublicHttpError if (HttpClientError.isHttpClientError(value)) { if (value.reason._tag === "TransportError") { - return isTimeoutException(value.reason.cause) ? TimeoutError : NetworkError + return isTimeoutException(value.reason.cause) ? TimeoutError : NetworkErrorBody } return value.reason._tag === "InvalidUrlError" ? InvalidUrlError : HttpRequestError } diff --git a/apps/web/src/lib/services/common/http-client.ts b/apps/web/src/lib/services/common/http-client.ts index 4ccb20e7a..10500ca29 100644 --- a/apps/web/src/lib/services/common/http-client.ts +++ b/apps/web/src/lib/services/common/http-client.ts @@ -2,6 +2,7 @@ import { FetchHttpClient } from "effect/unstable/http" import { Layer } from "effect" import { apiBaseUrl } from "./api-base-url" import { getMapleAuthHeaders } from "./auth-headers" +import { noteReachable, noteUnreachable, originOf } from "./peer-reachability" const CLIENT_TIMEOUT_MS = 45_000 @@ -23,13 +24,34 @@ const mapleFetch: typeof globalThis.fetch = async (input, init) => { } } - return globalThis.fetch(input, { - ...init, - headers, - signal: init?.signal ?? AbortSignal.timeout(CLIENT_TIMEOUT_MS), - }) + const origin = originOf(resolveRequestUrl(input)) + // Every API call the app makes passes through here, so this is where the app + // learns whether an origin is reachable at all — the same clock `tracedFetch` + // feeds from the ShapeStream side, since a blip takes both down at once. It + // only observes; `normalizeWarehouseError` is what reads it to decide whether + // a failure is the network's fault. An abort is evidence of nothing either + // way: we stopped listening. + return globalThis + .fetch(input, { + ...init, + headers, + signal: init?.signal ?? AbortSignal.timeout(CLIENT_TIMEOUT_MS), + }) + .then( + (response) => { + noteReachable(origin) + return response + }, + (cause: unknown) => { + if (!isAbort(cause)) noteUnreachable(origin, Date.now()) + throw cause + }, + ) } +const isAbort = (cause: unknown): boolean => + typeof cause === "object" && cause !== null && "name" in cause && cause.name === "AbortError" + export const MapleFetchHttpClientLive = FetchHttpClient.layer.pipe( Layer.provideMerge(Layer.succeed(FetchHttpClient.Fetch, mapleFetch)), ) diff --git a/apps/web/src/lib/services/common/otel-layer.ts b/apps/web/src/lib/services/common/otel-layer.ts index e21b7de9e..959ccd42b 100644 --- a/apps/web/src/lib/services/common/otel-layer.ts +++ b/apps/web/src/lib/services/common/otel-layer.ts @@ -27,7 +27,16 @@ const telemetry = MapleFlush.make({ }, // Expected 4xx API responses (the maple-web → maple-api edge surfaces these // as client-span failures) record as Ok instead of errors. - anticipatedErrorIdentifiers: [...ANTICIPATED_ERROR_IDENTIFIERS], + // + // `WarehouseUnreachableError` joins them as the one non-4xx member: a browser + // that briefly could not reach the API has not hit a fault worth + // fingerprinting, and the span still carries the failure. Only failures + // outlasting `PEER_OUTAGE_GRACE_MS` keep their reporting tag, so a genuine + // outage is unaffected — see `peer-reachability.ts`. + anticipatedErrorIdentifiers: [ + ...ANTICIPATED_ERROR_IDENTIFIERS, + "@maple/web/errors/WarehouseUnreachableError", + ], // rrweb self-recording. #225 disabled this while the recorder was pathological // (full-buffer re-stringify per flush, 30s DOM checkouts, unbounded buffer); // that same PR fixed all three (serialize-once at emit, 5-min checkouts, 4MB diff --git a/apps/web/src/lib/services/common/peer-reachability.test.ts b/apps/web/src/lib/services/common/peer-reachability.test.ts new file mode 100644 index 000000000..71bdb0f19 --- /dev/null +++ b/apps/web/src/lib/services/common/peer-reachability.test.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, it } from "vitest" + +import { + isBlipping, + noteReachable, + noteUnreachable, + originOf, + PEER_OUTAGE_GRACE_MS, +} from "./peer-reachability" + +const ORIGIN = "https://api.test" + +/** + * A wifi blip fails every request in flight at the same instant, which is why + * this measures elapsed time rather than counting failures: on the production + * session that fired the High error rate alert, four shape long-polls and every + * API call failed inside two seconds and the session then ran on for another 14 + * minutes. A counter would have escalated on the first of them. + */ +describe("peer reachability", () => { + beforeEach(() => { + noteReachable(ORIGIN) + noteReachable("https://other.test") + }) + + it("starts the clock at zero on the first failure", () => { + expect(noteUnreachable(ORIGIN, 1_000)).toBe(0) + }) + + it("measures from the first failure, not the previous one", () => { + noteUnreachable(ORIGIN, 1_000) + noteUnreachable(ORIGIN, 5_000) + + expect(noteUnreachable(ORIGIN, 9_000)).toBe(8_000) + }) + + it("restarts once the origin answers, so a later blip is a blip again", () => { + noteUnreachable(ORIGIN, 1_000) + noteReachable(ORIGIN) + + expect(noteUnreachable(ORIGIN, 60_000)).toBe(0) + }) + + it("tracks each origin separately", () => { + noteUnreachable(ORIGIN, 1_000) + + expect(noteUnreachable("https://other.test", 20_000)).toBe(0) + }) + + it("reads as blipping only inside the grace window", () => { + expect(isBlipping(ORIGIN, 1_000)).toBe(false) + + noteUnreachable(ORIGIN, 1_000) + + expect(isBlipping(ORIGIN, 1_000 + PEER_OUTAGE_GRACE_MS - 1)).toBe(true) + // Past the window the origin is not blipping — it is down, and a failure + // against it reports as an error again. + expect(isBlipping(ORIGIN, 1_000 + PEER_OUTAGE_GRACE_MS)).toBe(false) + }) + + it("stops reading as blipping once the origin answers", () => { + noteUnreachable(ORIGIN, 1_000) + noteReachable(ORIGIN) + + expect(isBlipping(ORIGIN, 1_001)).toBe(false) + }) +}) + +describe("originOf", () => { + it("drops the path and query so one clock covers a whole host", () => { + expect(originOf("https://api.test/internal/query-engine/execute-batch?x=1")).toBe("https://api.test") + }) + + it("falls back to the raw value when it is not a URL", () => { + expect(originOf("::::")).toBe("::::") + }) +}) diff --git a/apps/web/src/lib/services/common/peer-reachability.ts b/apps/web/src/lib/services/common/peer-reachability.ts new file mode 100644 index 000000000..41f131099 --- /dev/null +++ b/apps/web/src/lib/services/common/peer-reachability.ts @@ -0,0 +1,69 @@ +/** + * Whether the browser can currently reach a given origin. + * + * A rejection with no response at all — `TypeError: Failed to fetch`, an Effect + * `TransportError` — says the connection died, not that the application did. In + * a dashboard holding several Electric long-polls open around the clock, that + * happens on every wifi blip, VPN reconnect and laptop wake: at 16:55:39 on one + * production session, four shape long-polls and every in-flight API call failed + * inside two seconds while the *server* spans for those same requests completed + * `Ok` at 40s, and the session went on for another 14 minutes. Nothing failed; + * the browser stopped being able to reach anything. Reporting each one is what + * left maple-web at a 15% error rate and fired a critical High error rate alert. + * + * A blip and an outage differ in how long they last, not in what they throw, so + * that is what this measures. The first failure starts the clock, any response + * at all (a 500 included — the peer answered) stops it, and only failures still + * arriving after the grace window are treated as real. So a real outage — an + * unreachable API, a CORS misconfiguration, a bad base URL — still reports, + * continuously and from 15s in, while a blip reports nothing. + * + * Elapsed time, not a failure count: a blip fails every concurrent request at + * once, so counting would escalate on the first one. + * + * Keyed by origin, not by caller-facing peer name, because reachability is a + * property of the host: one clock covers the ShapeStream long-polls and the API + * calls that die in the same instant. + */ + +export const PEER_OUTAGE_GRACE_MS = 15_000 + +/** When each origin's current run of transport failures began. */ +const unreachableSince = new Map() + +/** The origin of `url`, or the whole string when it does not parse as one. */ +export const originOf = (url: string): string => { + try { + return new URL(url, typeof location === "undefined" ? undefined : location.href).origin + } catch { + return url + } +} + +/** + * Record a transport failure and answer how long the origin has been + * continuously unreachable — `0` for the failure that starts the run. + */ +export const noteUnreachable = (origin: string, now: number): number => { + const since = unreachableSince.get(origin) + if (since === undefined) { + unreachableSince.set(origin, now) + return 0 + } + return now - since +} + +/** The origin answered, so whatever run of failures preceded it is over. */ +export const noteReachable = (origin: string): void => { + unreachableSince.delete(origin) +} + +/** + * True while `origin` is inside a failure run that is still short enough to be a + * blip. Read-only — for a caller that holds a failure and needs to know whether + * to blame the network, without itself being new evidence. + */ +export const isBlipping = (origin: string, now: number): boolean => { + const since = unreachableSince.get(origin) + return since !== undefined && now - since < PEER_OUTAGE_GRACE_MS +} diff --git a/apps/web/src/lib/services/common/telemetry.ts b/apps/web/src/lib/services/common/telemetry.ts index a4800fb77..b30deca3b 100644 --- a/apps/web/src/lib/services/common/telemetry.ts +++ b/apps/web/src/lib/services/common/telemetry.ts @@ -1,4 +1,5 @@ import { Effect, Schema } from "effect" +import { noteReachable, noteUnreachable, PEER_OUTAGE_GRACE_MS } from "./peer-reachability" import { runtime } from "./runtime" const requestUrl = (input: RequestInfo | URL): string => @@ -18,8 +19,12 @@ const PAUSE_STREAM = "pause-stream" * Every ShapeStream fetch flows through `tracedFetch`, and Electric aborts them * routinely by design: `pause-stream` on pause/resume, a bare `AbortError` on * teardown. The server side of those traces completes `Ok` — nothing failed, the - * browser just stopped listening. Reporting them as span errors is what put - * maple-web at a ~15% error rate that was ~99% cancellations. + * browser just stopped listening, so reporting them as span errors inflated + * maple-web's error rate with cancellations. + * + * This covers only the aborts *we* issue. A connection that dies on its own + * rejects with a `TypeError`, which no abort signal explains and which this + * therefore declines; `PEER_OUTAGE_GRACE_MS` is what tells those apart. * * HTTP error responses are unaffected: a 5xx resolves the promise, so it never * reaches here and still lands on the `Error` path via `http.response.status_code`. @@ -121,6 +126,7 @@ export const tracedFetch = ( ), ) if (outcome.ok) { + noteReachable(parsed.origin) yield* Effect.annotateCurrentSpan( "http.response.status_code", outcome.response.status, @@ -130,16 +136,28 @@ export const tracedFetch = ( if (isCancellation(outcome.cause, init?.signal)) { // An abort is an expected outcome (navigation away, Electric // pause/resume), so the span stays `Ok` and only says what happened. + // The peer's reachability is untouched: we stopped listening, so + // the attempt is evidence of nothing either way. yield* Effect.annotateCurrentSpan({ "maple.http.cancelled": true, "error.type": "aborted", }) return outcome } - yield* Effect.annotateCurrentSpan( - "error.type", - causeName(outcome.cause) ?? "TracedFetchError", - ) + const unreachableMs = noteUnreachable(parsed.origin, Date.now()) + yield* Effect.annotateCurrentSpan({ + "error.type": causeName(outcome.cause) ?? "TracedFetchError", + "maple.http.unreachable_ms": unreachableMs, + }) + if (unreachableMs < PEER_OUTAGE_GRACE_MS) { + // Inside the grace window this is a connectivity blip, not a + // failure of the application: the span stays `Ok` and carries the + // annotations above, so the loss is still charted and alertable + // without fingerprinting an exception. The caller is rejected + // exactly as before and retries on its own. + yield* Effect.annotateCurrentSpan("maple.http.unreachable", true) + return outcome + } return yield* new TracedFetchError({ cause: outcome.cause, message: describeFetchFailure({ diff --git a/packages/effect-sdk/README.md b/packages/effect-sdk/README.md index 1439ba98b..e2de3d31c 100644 --- a/packages/effect-sdk/README.md +++ b/packages/effect-sdk/README.md @@ -79,7 +79,7 @@ When `MAPLE_INGEST_KEY` is unset, the SDK runs in no-op mode: buffers are draine | Option | Description | | ----------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `anticipatedErrorIdentifiers` | Stable `_tag` / `Error.name` identifiers for expected 4xx failures; exported as `Ok` without an exception | +| `anticipatedErrorIdentifiers` | Stable `_tag` / `Error.name` identifiers for expected 4xx failures; exported as `Ok` without an exception. A failure wrapped in an `{ error: … }` envelope is matched on the body's `_tag`, so an error decoded from an HTTP response classifies the same as the class that raised it | | `dropSpanNames` | Span names whose prefix matches an entry are dropped before OTLP export (e.g. `"McpServer/Notifications."`) | | `excludeLogSpans` | Skip Effect log spans in OTLP log attributes. Default `false` | | `tracesPath` | OTLP traces path appended to `endpoint`. Default `/v1/traces` | diff --git a/packages/effect-sdk/src/shared/flushable-tracer.test.ts b/packages/effect-sdk/src/shared/flushable-tracer.test.ts index 19ecf3523..48c36f0e6 100644 --- a/packages/effect-sdk/src/shared/flushable-tracer.test.ts +++ b/packages/effect-sdk/src/shared/flushable-tracer.test.ts @@ -105,6 +105,48 @@ describe("makeSpanBuffer anticipated-error classification", () => { }), ) + // The shape a decoded HTTP error body actually has on the client: an `{ error }` + // envelope, not a class. Without the unwrap an API using that convention + // matches *no* configured identifier at all, and every expected 4xx records + // `Error` with the stringified envelope as its whole message. + it.effect("classifies a decoded `{ error: { _tag } }` envelope by the body's tag", () => + Effect.gen(function* () { + const buffer = makeSpanBuffer({ + anticipatedErrorIdentifiers: new Set(["@maple/http/v2/SessionReplayRangeTooLargeError"]), + }) + yield* runSpan( + buffer, + Effect.fail({ + error: { + _tag: "@maple/http/v2/SessionReplayRangeTooLargeError", + type: "invalid_request_error", + code: "range_too_large", + message: "That part of the recording is too large to load in one request.", + }, + }), + ) + const [span] = buffer.drain() + assert.isDefined(span) + assert.strictEqual(span!.status.code, 1 /* Ok */) + assert.strictEqual( + span!.events.some((event) => event.name === "exception"), + false, + ) + }), + ) + + it.effect("leaves an envelope whose tag is not anticipated an Error span", () => + Effect.gen(function* () { + const buffer = makeSpanBuffer({ + anticipatedErrorIdentifiers: new Set(["@maple/http/v2/SessionReplayRangeTooLargeError"]), + }) + yield* runSpan(buffer, Effect.fail({ error: { _tag: "@maple/http/errors/PersistenceError" } })) + const [span] = buffer.drain() + assert.isDefined(span) + assert.strictEqual(span!.status.code, 2 /* Error */) + }), + ) + it.effect("still marks an unclassified failure as an Error span with an exception event", () => Effect.gen(function* () { const buffer = makeSpanBuffer({ anticipatedErrorTags: tags }) diff --git a/packages/effect-sdk/src/shared/flushable-tracer.ts b/packages/effect-sdk/src/shared/flushable-tracer.ts index cab1d594f..d27e8e9dc 100644 --- a/packages/effect-sdk/src/shared/flushable-tracer.ts +++ b/packages/effect-sdk/src/shared/flushable-tracer.ts @@ -216,6 +216,15 @@ const generateId = (len: number): string => { const failureIdentifier = (error: unknown): string | undefined => { if (Predicate.hasProperty(error, "_tag") && typeof error._tag === "string") return error._tag if (Predicate.hasProperty(error, "name") && typeof error.name === "string") return error.name + // An error that crossed an HTTP boundary arrives as a decoded *body*, not as + // the class that raised it. An API that wraps its bodies in `{ error: … }` — + // a common envelope convention — therefore hands the failure channel a plain + // object with no identifier of its own, and every identifier a caller + // configured goes unmatched: expected 4xx answers record as `Error` spans + // whose entire message is the JSON-stringified envelope. Unwrap one level, and + // only for the body's own tag. + const body = Predicate.hasProperty(error, "error") ? error.error : undefined + if (Predicate.hasProperty(body, "_tag") && typeof body._tag === "string") return body._tag return undefined }