diff --git a/CHANGES.md b/CHANGES.md index c57a58599..f691c4e63 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -41,13 +41,25 @@ To be released. attributes, and `TraceActivityRecord.activityJson` is present only when the span event includes full activity JSON. [[#316], [#619], [#755]] + - Added OpenTelemetry HTTP server metrics for inbound requests handled by + `Federation.fetch()`: `fedify.http.server.request.count` (Counter) and + `fedify.http.server.request.duration` (Histogram). Both instruments carry + `http.request.method`, `fedify.endpoint`, optional + `http.response.status_code`, and optional `fedify.route.template` + attributes so that operators can monitor aggregate request rate, latency, + and status-code error rate even when traces are sampled. Attributes + deliberately exclude raw URLs, query strings, and identifier values to + keep cardinality bounded. [[#316], [#736], [#757]] + [#316]: https://github.com/fedify-dev/fedify/issues/316 [#619]: https://github.com/fedify-dev/fedify/issues/619 [#735]: https://github.com/fedify-dev/fedify/issues/735 +[#736]: https://github.com/fedify-dev/fedify/issues/736 [#748]: https://github.com/fedify-dev/fedify/pull/748 [#752]: https://github.com/fedify-dev/fedify/issues/752 [#753]: https://github.com/fedify-dev/fedify/pull/753 [#755]: https://github.com/fedify-dev/fedify/pull/755 +[#757]: https://github.com/fedify-dev/fedify/pull/757 ### @fedify/fixture diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index ff81f27dc..e6288ba9c 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -296,13 +296,15 @@ Instrumented metrics Fedify records the following OpenTelemetry metrics: -| Metric name | Instrument | Unit | Description | -| -------------------------------------------- | ---------- | ----------- | ----------------------------------------------------------- | -| `activitypub.delivery.sent` | Counter | `{attempt}` | Counts outgoing ActivityPub delivery attempts. | -| `activitypub.delivery.permanent_failure` | Counter | `{failure}` | Counts outgoing deliveries abandoned as permanent failures. | -| `activitypub.delivery.duration` | Histogram | `ms` | Measures outgoing ActivityPub delivery attempt duration. | -| `activitypub.inbox.processing_duration` | Histogram | `ms` | Measures inbox listener processing duration. | -| `activitypub.signature.verification_failure` | Counter | `{failure}` | Counts failed signature verification for inbox requests. | +| Metric name | Instrument | Unit | Description | +| -------------------------------------------- | ---------- | ----------- | --------------------------------------------------------------- | +| `activitypub.delivery.sent` | Counter | `{attempt}` | Counts outgoing ActivityPub delivery attempts. | +| `activitypub.delivery.permanent_failure` | Counter | `{failure}` | Counts outgoing deliveries abandoned as permanent failures. | +| `activitypub.delivery.duration` | Histogram | `ms` | Measures outgoing ActivityPub delivery attempt duration. | +| `activitypub.inbox.processing_duration` | Histogram | `ms` | Measures inbox listener processing duration. | +| `activitypub.signature.verification_failure` | Counter | `{failure}` | Counts failed signature verification for inbox requests. | +| `fedify.http.server.request.count` | Counter | `{request}` | Counts inbound HTTP requests handled by `Federation.fetch()`. | +| `fedify.http.server.request.duration` | Histogram | `ms` | Measures inbound HTTP request duration in `Federation.fetch()`. | ### Metric attributes @@ -324,11 +326,42 @@ Fedify records the following OpenTelemetry metrics: : `activitypub.verification.failure_reason`, plus `activitypub.remote.host` when the failed signature includes a key ID. +`fedify.http.server.request.count` and `fedify.http.server.request.duration` +: `http.request.method` and `fedify.endpoint` are always present. + `http.request.method` is normalized to one of the standard HTTP methods + (`CONNECT`, `DELETE`, `GET`, `HEAD`, `OPTIONS`, `PATCH`, `POST`, `PUT`, + `QUERY`, `TRACE`) or `_OTHER` for any other value, so that an arbitrary + client cannot inflate metric cardinality by sending custom methods. + `http.response.status_code` is recorded when a `Response` is produced + (success and non-2xx alike) and omitted when the request threw an + exception before a response could be returned. `fedify.route.template` + is recorded when a route matched, and contains the [URI Template] + parameter names (for example `/users/{identifier}`) rather than the + matched parameter values. + Fedify records `activitypub.remote.host` as the URL hostname only; ports, paths, and query strings are deliberately excluded to keep metric cardinality bounded. Activity types use the same qualified URI form as Fedify's trace attributes, for example `https://www.w3.org/ns/activitystreams#Create`. +The HTTP server request metrics deliberately exclude high-cardinality fields +such as the full URL, raw path, query string, actor identifier, and inbox +URL. Use the request span's `url.full` attribute when you need the exact URL +for a sampled trace; the metrics expose the stable endpoint category and route +template so that aggregate request rate, latency, and status-code error rate +remain meaningful even when traces are sampled. + +The `fedify.endpoint` attribute is drawn from a fixed enumeration: +`webfinger`, `nodeinfo`, `actor`, `inbox`, `shared_inbox`, `outbox`, +`object`, `following`, `followers`, `liked`, `featured`, `featured_tags`, +`collection`, `not_found`, `not_acceptable`, and `error`. When a request +throws an exception after Fedify has already classified its endpoint, the +metric retains the matched endpoint (for example `actor`) so that +fault-attribution stays per endpoint; `error` is only used when classification +itself failed. + +[URI Template]: https://datatracker.ietf.org/doc/html/rfc6570 + Semantic [attributes] for ActivityPub ------------------------------------- @@ -367,6 +400,8 @@ for ActivityPub: | `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | | `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | | `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | +| `fedify.endpoint` | string | The bounded endpoint category that classified an inbound HTTP request handled by `Federation.fetch()`. | `"actor"` | +| `fedify.route.template` | string | The matched URI Template, with parameter names (not values). | `"/users/{identifier}"` | | `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | | `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | | `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index 8399b375b..2decbc9c9 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -13,6 +13,8 @@ class FederationMetrics { readonly signatureVerificationFailure: Counter; readonly deliveryDuration: Histogram; readonly inboxProcessingDuration: Histogram; + readonly httpServerRequestCount: Counter; + readonly httpServerRequestDuration: Histogram; constructor(meterProvider: MeterProvider) { const meter = meterProvider.getMeter(metadata.name, metadata.version); @@ -48,6 +50,40 @@ class FederationMetrics { unit: "ms", }, ); + this.httpServerRequestCount = meter.createCounter( + "fedify.http.server.request.count", + { + description: "HTTP requests handled by Federation.fetch().", + unit: "{request}", + }, + ); + this.httpServerRequestDuration = meter.createHistogram( + "fedify.http.server.request.duration", + { + description: "Duration of HTTP requests handled by Federation.fetch().", + unit: "ms", + advice: { + // Mirror the OpenTelemetry HTTP server semantic-conventions + // recommended buckets, expressed in milliseconds. + explicitBucketBoundaries: [ + 5, + 10, + 25, + 50, + 75, + 100, + 250, + 500, + 750, + 1000, + 2500, + 5000, + 7500, + 10000, + ], + }, + }, + ); } recordDelivery( @@ -95,6 +131,44 @@ class FederationMetrics { "activitypub.activity.type": activityType, }); } + + recordHttpServerRequest( + method: string, + endpoint: string, + durationMs: number, + options: { statusCode?: number; routeTemplate?: string } = {}, + ): void { + const attributes: Attributes = { + "http.request.method": normalizeHttpMethod(method), + "fedify.endpoint": endpoint, + }; + if (options.statusCode != null) { + attributes["http.response.status_code"] = options.statusCode; + } + if (options.routeTemplate != null) { + attributes["fedify.route.template"] = options.routeTemplate; + } + this.httpServerRequestCount.add(1, attributes); + this.httpServerRequestDuration.record(durationMs, attributes); + } +} + +const KNOWN_HTTP_METHODS: ReadonlySet = new Set([ + "CONNECT", + "DELETE", + "GET", + "HEAD", + "OPTIONS", + "PATCH", + "POST", + "PUT", + "QUERY", + "TRACE", +]); + +function normalizeHttpMethod(method: string): string { + const upper = method.toUpperCase(); + return KNOWN_HTTP_METHODS.has(upper) ? upper : "_OTHER"; } const federationMetrics = new WeakMap(); diff --git a/packages/fedify/src/federation/middleware.test.ts b/packages/fedify/src/federation/middleware.test.ts index 681a52805..7de8556cc 100644 --- a/packages/fedify/src/federation/middleware.test.ts +++ b/packages/fedify/src/federation/middleware.test.ts @@ -1455,6 +1455,370 @@ test("Federation.fetch()", async (t) => { fetchMock.hardReset(); }); +test("Federation.fetch() records HTTP server request metrics", async (t) => { + const createTestContext = () => { + const kv = new MemoryKvStore(); + const [meterProvider, recorder] = createTestMeterProvider(); + const federation = createFederation({ + kv, + meterProvider, + documentLoaderFactory: () => mockDocumentLoader, + }); + + federation.setActorDispatcher( + "/users/{identifier}", + (ctx, identifier) => { + if (identifier === "boom") { + throw new Error("explosion in actor dispatcher"); + } + return new vocab.Person({ + id: ctx.getActorUri(identifier), + inbox: ctx.getInboxUri(identifier), + preferredUsername: identifier, + }); + }, + ); + + federation.setNodeInfoDispatcher("/nodeinfo/2.1", () => ({ + software: { name: "example", version: "1.0.0" }, + protocols: ["activitypub"], + usage: { users: {}, localPosts: 0, localComments: 0 }, + })); + + federation.setFollowersDispatcher( + "/users/{identifier}/followers", + () => ({ items: [] }), + ); + + federation.setCollectionDispatcher( + "custom-collection", + vocab.Object, + "/users/{identifier}/custom/{id}", + () => ({ items: [] }), + ); + + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox"); + + return { federation, recorder }; + }; + + await t.step("records a successful actor request", async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/users/alice", { + method: "GET", + headers: { "Accept": "application/activity+json" }, + }), + { contextData: undefined }, + ); + assertEquals(response.status, 200); + + const counts = recorder.getMeasurements("fedify.http.server.request.count"); + assertEquals(counts.length, 1); + assertEquals(counts[0].type, "counter"); + assertEquals(counts[0].value, 1); + assertEquals(counts[0].attributes["http.request.method"], "GET"); + assertEquals(counts[0].attributes["fedify.endpoint"], "actor"); + assertEquals(counts[0].attributes["http.response.status_code"], 200); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/users/{identifier}", + ); + + const durations = recorder.getMeasurements( + "fedify.http.server.request.duration", + ); + assertEquals(durations.length, 1); + assertEquals(durations[0].type, "histogram"); + assert(durations[0].value >= 0); + assertEquals(durations[0].attributes["fedify.endpoint"], "actor"); + assertEquals(durations[0].attributes["http.response.status_code"], 200); + assertEquals( + durations[0].attributes["fedify.route.template"], + "/users/{identifier}", + ); + }); + + await t.step("records WebFinger requests", async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request( + "https://example.com/.well-known/webfinger?resource=acct:alice@example.com", + ), + { contextData: undefined }, + ); + assertEquals(response.status, 200); + + const counts = recorder.getMeasurements("fedify.http.server.request.count"); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "webfinger"); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/.well-known/webfinger", + ); + assertEquals(counts[0].attributes["http.response.status_code"], 200); + }); + + await t.step("records NodeInfo JRD requests", async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/.well-known/nodeinfo"), + { contextData: undefined }, + ); + assertEquals(response.status, 200); + + const counts = recorder.getMeasurements("fedify.http.server.request.count"); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "nodeinfo"); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/.well-known/nodeinfo", + ); + }); + + await t.step("records NodeInfo dispatcher requests", async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/nodeinfo/2.1"), + { contextData: undefined }, + ); + assertEquals(response.status, 200); + + const counts = recorder.getMeasurements("fedify.http.server.request.count"); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "nodeinfo"); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/nodeinfo/2.1", + ); + }); + + await t.step("records 404 not_found for unmatched paths", async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/no/such/path"), + { contextData: undefined }, + ); + assertEquals(response.status, 404); + + const counts = recorder.getMeasurements("fedify.http.server.request.count"); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "not_found"); + assertEquals(counts[0].attributes["http.response.status_code"], 404); + assertEquals(counts[0].attributes["fedify.route.template"], undefined); + }); + + await t.step( + "records 406 not_acceptable when JSON-LD Accept missing", + async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/users/alice", { + method: "GET", + headers: { "Accept": "text/html" }, + }), + { contextData: undefined }, + ); + assertEquals(response.status, 406); + + const counts = recorder.getMeasurements( + "fedify.http.server.request.count", + ); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "not_acceptable"); + assertEquals(counts[0].attributes["http.response.status_code"], 406); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/users/{identifier}", + ); + }, + ); + + await t.step( + "records thrown errors after classification with the matched endpoint", + async () => { + const { federation, recorder } = createTestContext(); + await assertRejects( + () => + federation.fetch( + new Request("https://example.com/users/boom", { + method: "GET", + headers: { "Accept": "application/activity+json" }, + }), + { contextData: undefined }, + ), + Error, + "explosion", + ); + + const counts = recorder.getMeasurements( + "fedify.http.server.request.count", + ); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "actor"); + assertEquals( + counts[0].attributes["http.response.status_code"], + undefined, + ); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/users/{identifier}", + ); + + const durations = recorder.getMeasurements( + "fedify.http.server.request.duration", + ); + assertEquals(durations.length, 1); + assertEquals(durations[0].attributes["fedify.endpoint"], "actor"); + }, + ); + + await t.step( + "collapses user-defined collection dispatchers to endpoint=collection", + async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/users/alice/custom/1", { + method: "GET", + headers: { "Accept": "application/activity+json" }, + }), + { contextData: undefined }, + ); + assertEquals(response.status, 200); + + const counts = recorder.getMeasurements( + "fedify.http.server.request.count", + ); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "collection"); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/users/{identifier}/custom/{id}", + ); + }, + ); + + await t.step("records followers as endpoint=followers", async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/users/alice/followers", { + method: "GET", + headers: { "Accept": "application/activity+json" }, + }), + { contextData: undefined }, + ); + assertEquals(response.status, 200); + + const counts = recorder.getMeasurements("fedify.http.server.request.count"); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "followers"); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/users/{identifier}/followers", + ); + }); + + await t.step("records sharedInbox as endpoint=shared_inbox", async () => { + const kv = new MemoryKvStore(); + const [meterProvider, recorder] = createTestMeterProvider(); + const federation = createFederation({ + kv, + meterProvider, + documentLoaderFactory: () => mockDocumentLoader, + }); + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox"); + + const response = await federation.fetch( + new Request("https://example.com/inbox", { + method: "POST", + headers: { "accept": "application/ld+json" }, + }), + { contextData: undefined }, + ); + // Without an actor dispatcher signature verification fails — but the + // routing classification has already happened, which is what we assert. + assert(response.status >= 400); + + const counts = recorder.getMeasurements("fedify.http.server.request.count"); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["fedify.endpoint"], "shared_inbox"); + assertEquals( + counts[0].attributes["fedify.route.template"], + "/inbox", + ); + assertEquals(counts[0].attributes["http.request.method"], "POST"); + }); + + await t.step( + "normalizes unknown HTTP methods to _OTHER for cardinality control", + async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/users/alice", { + method: "PROPFIND", + headers: { "Accept": "application/activity+json" }, + }), + { contextData: undefined }, + ); + // We only care about the metric attribute, not the response code here. + assert(response.status >= 100); + + const counts = recorder.getMeasurements( + "fedify.http.server.request.count", + ); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["http.request.method"], "_OTHER"); + }, + ); + + await t.step( + "preserves QUERY as a known HTTP method", + async () => { + const { federation, recorder } = createTestContext(); + const response = await federation.fetch( + new Request("https://example.com/users/alice", { + method: "QUERY", + headers: { "Accept": "application/activity+json" }, + }), + { contextData: undefined }, + ); + assert(response.status >= 100); + + const counts = recorder.getMeasurements( + "fedify.http.server.request.count", + ); + assertEquals(counts.length, 1); + assertEquals(counts[0].attributes["http.request.method"], "QUERY"); + }, + ); + + await t.step( + "uses the global meter provider when none is configured", + async () => { + const kv = new MemoryKvStore(); + const federation = createFederation({ + kv, + documentLoaderFactory: () => mockDocumentLoader, + }); + federation.setActorDispatcher( + "/users/{identifier}", + (ctx, identifier) => + new vocab.Person({ id: ctx.getActorUri(identifier) }), + ); + + // Should not throw — the no-op meter provider absorbs the calls. + const response = await federation.fetch( + new Request("https://example.com/users/alice", { + method: "GET", + headers: { "Accept": "application/activity+json" }, + }), + { contextData: undefined }, + ); + assertEquals(response.status, 200); + }, + ); +}); + test("Federation.setInboxListeners()", async (t) => { const kv = new MemoryKvStore(); diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 3ccbcba1a..f29e3a84c 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -1366,6 +1366,8 @@ export class FederationImpl const requestId = getRequestId(request); return withContext({ requestId }, async () => { const tracer = this._getTracer(); + const metricState: HttpMetricState = {}; + const metricStart = performance.now(); return await tracer.startActiveSpan( request.method, { @@ -1392,11 +1394,19 @@ export class FederationImpl ...options, span, tracer, + metricState, }); if (acceptsJsonLd(request)) { response.headers.set("Vary", "Accept"); } } catch (error) { + getFederationMetrics(this.meterProvider) + .recordHttpServerRequest( + request.method, + metricState.endpoint ?? "error", + getDurationMs(metricStart), + { routeTemplate: metricState.routeTemplate }, + ); span.setStatus({ code: SpanStatusCode.ERROR, message: `${error}`, @@ -1409,6 +1419,15 @@ export class FederationImpl ); throw error; } + getFederationMetrics(this.meterProvider).recordHttpServerRequest( + request.method, + metricState.endpoint ?? "error", + getDurationMs(metricStart), + { + statusCode: response.status, + routeTemplate: metricState.routeTemplate, + }, + ); if (span.isRecording()) { span.setAttribute( ATTR_HTTP_RESPONSE_STATUS_CODE, @@ -1453,14 +1472,24 @@ export class FederationImpl contextData, span, tracer, - }: FederationFetchOptions & { span: Span; tracer: Tracer }, + metricState, + }: FederationFetchOptions & { + span: Span; + tracer: Tracer; + metricState: HttpMetricState; + }, ): Promise { onNotFound ??= notFound; onNotAcceptable ??= notAcceptable; onUnauthorized ??= unauthorized; const url = new URL(request.url); const route = this.router.route(url.pathname); - if (route == null) return await onNotFound(request); + if (route == null) { + metricState.endpoint = "not_found"; + return await onNotFound(request); + } + metricState.routeTemplate = route.template; + metricState.endpoint = getEndpointCategory(route.name); span.updateName(`${request.method} ${route.template}`); let context = this.#createContext(request, contextData); const routeName = route.name.replace(/:.*$/, ""); @@ -1489,6 +1518,7 @@ export class FederationImpl // Routes that require JSON-LD Accepts header: if (request.method !== "POST" && !acceptsJsonLd(request)) { + metricState.endpoint = "not_acceptable"; return await onNotAcceptable(request); } switch (routeName) { @@ -1724,6 +1754,7 @@ export class FederationImpl }); } default: { + metricState.endpoint = "not_found"; const response = onNotFound(request); return response instanceof Promise ? await response : response; } @@ -1731,6 +1762,67 @@ export class FederationImpl } } +type FedifyEndpoint = + | "webfinger" + | "nodeinfo" + | "actor" + | "inbox" + | "shared_inbox" + | "outbox" + | "object" + | "following" + | "followers" + | "liked" + | "featured" + | "featured_tags" + | "collection" + | "not_found" + | "not_acceptable" + | "error"; + +interface HttpMetricState { + endpoint?: FedifyEndpoint; + routeTemplate?: string; +} + +function getEndpointCategory(routeName: string): FedifyEndpoint { + if (routeName.startsWith("object:")) return "object"; + if ( + routeName.startsWith("collection:") || + routeName.startsWith("orderedCollection:") + ) { + return "collection"; + } + if (routeName.startsWith(ACTOR_ALIAS_PREFIX)) return "actor"; + switch (routeName) { + case "webfinger": + return "webfinger"; + case "nodeInfoJrd": + case "nodeInfo": + return "nodeinfo"; + case "actor": + return "actor"; + case "inbox": + return "inbox"; + case "sharedInbox": + return "shared_inbox"; + case "outbox": + return "outbox"; + case "following": + return "following"; + case "followers": + return "followers"; + case "liked": + return "liked"; + case "featured": + return "featured"; + case "featuredTags": + return "featured_tags"; + default: + return "not_found"; + } +} + interface ContextOptions { url: URL; federation: FederationImpl;