From 0015faf969d45fbfc6d956bdab7a09f2daa7be37 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 5 May 2026 20:00:05 +0900 Subject: [PATCH 01/16] Add ActivityPub federation metrics Record OpenTelemetry metrics for delivery attempts, permanent delivery failures, signature verification failures, and inbox listener processing. Thread the meter provider through federation contexts so applications and tests can choose a provider explicitly. Refs: https://github.com/fedify-dev/fedify/issues/619 Assisted-by: gpt-5.5 --- deno.json | 1 + deno.lock | 40 ++- packages/fedify/package.json | 1 + packages/fedify/src/federation/context.ts | 8 +- packages/fedify/src/federation/federation.ts | 9 +- .../fedify/src/federation/handler.test.ts | 50 +++- packages/fedify/src/federation/handler.ts | 13 + packages/fedify/src/federation/inbox.ts | 35 ++- packages/fedify/src/federation/metrics.ts | 133 +++++++++ .../fedify/src/federation/middleware.test.ts | 106 +++++++- packages/fedify/src/federation/middleware.ts | 59 +++- packages/fedify/src/federation/send.test.ts | 152 +++++++++++ packages/fedify/src/federation/send.ts | 93 ++++--- packages/fedify/src/testing/context.ts | 4 +- packages/fixture/src/otel.ts | 254 ++++++++++++++++++ packages/testing/src/context.ts | 25 ++ packages/testing/src/mock.ts | 31 +++ pnpm-lock.yaml | 6 + pnpm-workspace.yaml | 1 + 19 files changed, 961 insertions(+), 60 deletions(-) create mode 100644 packages/fedify/src/federation/metrics.ts diff --git a/deno.json b/deno.json index f8495b6cb..55f99c3a7 100644 --- a/deno.json +++ b/deno.json @@ -47,6 +47,7 @@ "@opentelemetry/api": "npm:@opentelemetry/api@^1.9.0", "@opentelemetry/context-async-hooks": "npm:@opentelemetry/context-async-hooks@^2.5.0", "@opentelemetry/core": "npm:@opentelemetry/core@^2.5.0", + "@opentelemetry/sdk-metrics": "npm:@opentelemetry/sdk-metrics@2.5.0", "@opentelemetry/sdk-trace-base": "npm:@opentelemetry/sdk-trace-base@^2.5.0", "@opentelemetry/semantic-conventions": "npm:@opentelemetry/semantic-conventions@^1.39.0", "@optique/config": "jsr:@optique/config@^1.0.2", diff --git a/deno.lock b/deno.lock index d1ae8395f..6d41fb003 100644 --- a/deno.lock +++ b/deno.lock @@ -84,7 +84,8 @@ "npm:@nuxt/schema@4": "4.4.2", "npm:@opentelemetry/api@^1.9.0": "1.9.1", "npm:@opentelemetry/context-async-hooks@^2.5.0": "2.7.0_@opentelemetry+api@1.9.1", - "npm:@opentelemetry/core@^2.5.0": "2.7.0_@opentelemetry+api@1.9.1", + "npm:@opentelemetry/core@^2.5.0": "2.7.1_@opentelemetry+api@1.9.1", + "npm:@opentelemetry/sdk-metrics@2.5.0": "2.5.0_@opentelemetry+api@1.9.1", "npm:@opentelemetry/sdk-trace-base@^2.5.0": "2.7.0_@opentelemetry+api@1.9.1", "npm:@opentelemetry/semantic-conventions@^1.39.0": "1.40.0", "npm:@poppanator/http-constants@^1.1.1": "1.1.1", @@ -2451,6 +2452,13 @@ "@opentelemetry/api" ] }, + "@opentelemetry/core@2.5.0_@opentelemetry+api@1.9.1": { + "integrity": "sha512-ka4H8OM6+DlUhSAZpONu0cPBtPPTQKxbxVzC4CzVx5+K4JnroJVBtDzLAMx4/3CDTJXRvVFhpFjtl4SaiTNoyQ==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/semantic-conventions" + ] + }, "@opentelemetry/core@2.7.0_@opentelemetry+api@1.9.1": { "integrity": "sha512-DT12SXVwV2eoJrGf4nnsvZojxxeQo+LlNAsoYGRRObPWTeN6APiqZ2+nqDCQDvQX40eLi1AePONS0onoASp3yQ==", "dependencies": [ @@ -2458,20 +2466,43 @@ "@opentelemetry/semantic-conventions" ] }, + "@opentelemetry/core@2.7.1_@opentelemetry+api@1.9.1": { + "integrity": "sha512-QAqIj32AtK6+pEVNG7EOVxHdE06RP+FM5qpiEJ4RtDcFIqKUZHYhl7/7UY5efhwmwNAg7j8QbJVBLxMerc0+gw==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/semantic-conventions" + ] + }, + "@opentelemetry/resources@2.5.0_@opentelemetry+api@1.9.1": { + "integrity": "sha512-F8W52ApePshpoSrfsSk1H2yJn9aKjCrbpQF1M9Qii0GHzbfVeFUB+rc3X4aggyZD8x9Gu3Slua+s6krmq6Dt8g==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.5.0_@opentelemetry+api@1.9.1", + "@opentelemetry/semantic-conventions" + ] + }, "@opentelemetry/resources@2.7.0_@opentelemetry+api@1.9.1": { "integrity": "sha512-K+oi0hNMv94EpZbnW3eyu2X6SGVpD3O5DhG2NIp65Hc7lhAj9brRXTAVzh3wB82+q3ThakEf7Zd7RsFUqcTc7A==", "dependencies": [ "@opentelemetry/api", - "@opentelemetry/core", + "@opentelemetry/core@2.7.0_@opentelemetry+api@1.9.1", "@opentelemetry/semantic-conventions" ] }, + "@opentelemetry/sdk-metrics@2.5.0_@opentelemetry+api@1.9.1": { + "integrity": "sha512-BeJLtU+f5Gf905cJX9vXFQorAr6TAfK3SPvTFqP+scfIpDQEJfRaGJWta7sJgP+m4dNtBf9y3yvBKVAZZtJQVA==", + "dependencies": [ + "@opentelemetry/api", + "@opentelemetry/core@2.5.0_@opentelemetry+api@1.9.1", + "@opentelemetry/resources@2.5.0_@opentelemetry+api@1.9.1" + ] + }, "@opentelemetry/sdk-trace-base@2.7.0_@opentelemetry+api@1.9.1": { "integrity": "sha512-Yg9zEXJB50DLVLpsKPk7NmNqlPlS+OvqhJGh0A8oawIOTPOwlm4eXs9BMJV7L79lvEwI+dWtAj+YjTyddV336A==", "dependencies": [ "@opentelemetry/api", - "@opentelemetry/core", - "@opentelemetry/resources", + "@opentelemetry/core@2.7.0_@opentelemetry+api@1.9.1", + "@opentelemetry/resources@2.7.0_@opentelemetry+api@1.9.1", "@opentelemetry/semantic-conventions" ] }, @@ -9269,6 +9300,7 @@ "npm:@opentelemetry/api@^1.9.0", "npm:@opentelemetry/context-async-hooks@^2.5.0", "npm:@opentelemetry/core@^2.5.0", + "npm:@opentelemetry/sdk-metrics@2.5.0", "npm:@opentelemetry/sdk-trace-base@^2.5.0", "npm:@opentelemetry/semantic-conventions@^1.39.0", "npm:@solidjs/start@^1.3.0", diff --git a/packages/fedify/package.json b/packages/fedify/package.json index 8efbc46ad..4aab14153 100644 --- a/packages/fedify/package.json +++ b/packages/fedify/package.json @@ -160,6 +160,7 @@ "devDependencies": { "@fedify/fixture": "workspace:*", "@fedify/vocab-tools": "workspace:^", + "@opentelemetry/sdk-metrics": "catalog:", "@std/assert": "jsr:^0.226.0", "@std/path": "catalog:", "@types/node": "^24.2.1", diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index 0a82afdcf..ec6eafd30 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -16,7 +16,7 @@ import type { LookupWebFingerOptions, ResourceDescriptor, } from "@fedify/webfinger"; -import type { TracerProvider } from "@opentelemetry/api"; +import type { MeterProvider, TracerProvider } from "@opentelemetry/api"; import type { GetNodeInfoOptions } from "../nodeinfo/client.ts"; import type { JsonValue, NodeInfo } from "../nodeinfo/types.ts"; import type { GetKeyOwnerOptions } from "../sig/owner.ts"; @@ -70,6 +70,12 @@ export interface Context { */ readonly tracerProvider: TracerProvider; + /** + * The OpenTelemetry meter provider. + * @since 2.3.0 + */ + readonly meterProvider: MeterProvider; + /** * The document loader for loading remote JSON-LD documents. */ diff --git a/packages/fedify/src/federation/federation.ts b/packages/fedify/src/federation/federation.ts index fa91f2253..483d0b764 100644 --- a/packages/fedify/src/federation/federation.ts +++ b/packages/fedify/src/federation/federation.ts @@ -10,7 +10,7 @@ import type { DocumentLoaderFactory, GetUserAgentOptions, } from "@fedify/vocab-runtime"; -import type { TracerProvider } from "@opentelemetry/api"; +import type { MeterProvider, TracerProvider } from "@opentelemetry/api"; import type { ActivityTransformer } from "../compat/types.ts"; import type { HttpMessageSignaturesSpec } from "../sig/http.ts"; import type { @@ -1052,6 +1052,13 @@ export interface FederationOptions { * @since 1.3.0 */ tracerProvider?: TracerProvider; + + /** + * The OpenTelemetry meter provider for recording metrics. If not provided, + * the default global meter provider is used. + * @since 2.3.0 + */ + meterProvider?: MeterProvider; } /** diff --git a/packages/fedify/src/federation/handler.test.ts b/packages/fedify/src/federation/handler.test.ts index 9bf81cc48..ec94351f4 100644 --- a/packages/fedify/src/federation/handler.test.ts +++ b/packages/fedify/src/federation/handler.test.ts @@ -1,4 +1,5 @@ import { + createTestMeterProvider, createTestTracerProvider, mockDocumentLoader, test, @@ -12,7 +13,12 @@ import { Tombstone, } from "@fedify/vocab"; import { FetchError } from "@fedify/vocab-runtime"; -import { assert, assertEquals, assertInstanceOf } from "@std/assert"; +import { + assert, + assertEquals, + assertGreaterOrEqual, + assertInstanceOf, +} from "@std/assert"; import { parseAcceptSignature } from "../sig/accept.ts"; import { signRequest } from "../sig/http.ts"; import { @@ -2318,8 +2324,13 @@ test("handleCustomCollection()", async () => { test("handleInbox() records OpenTelemetry span events", async () => { const [tracerProvider, exporter] = createTestTracerProvider(); + const [meterProvider, recorder] = createTestMeterProvider(); const kv = new MemoryKvStore(); - const federation = createFederation({ kv, tracerProvider }); + const federation = createFederation({ + kv, + meterProvider, + tracerProvider, + }); const activity = new Create({ id: new URL("https://example.com/activity"), @@ -2390,6 +2401,7 @@ test("handleInbox() records OpenTelemetry span events", async () => { onNotFound: (_request) => new Response("Not found", { status: 404 }), signatureTimeWindow: false, skipSignatureVerification: true, + meterProvider, tracerProvider, }); @@ -2428,12 +2440,28 @@ test("handleInbox() records OpenTelemetry span events", async () => { ); assertEquals(recordedActivity.id, "https://example.com/activity"); assertEquals(recordedActivity.type, "Create"); + + const durations = recorder.getMeasurements( + "activitypub.inbox.processing_duration", + ); + assertEquals(durations.length, 1); + assertEquals(durations[0].type, "histogram"); + assertGreaterOrEqual(durations[0].value, 0); + assertEquals( + durations[0].attributes["activitypub.activity.type"], + "https://www.w3.org/ns/activitystreams#Create", + ); }); test("handleInbox() records unverified HTTP signature details", async () => { const [tracerProvider, exporter] = createTestTracerProvider(); + const [meterProvider, recorder] = createTestMeterProvider(); const kv = new MemoryKvStore(); - const federation = createFederation({ kv, tracerProvider }); + const federation = createFederation({ + kv, + meterProvider, + tracerProvider, + }); const keyId = new URL("https://gone.example/users/someone#main-key"); const activity = new Create({ @@ -2508,6 +2536,7 @@ test("handleInbox() records unverified HTTP signature details", async () => { onNotFound: (_request) => new Response("Not found", { status: 404 }), signatureTimeWindow: false, skipSignatureVerification: false, + meterProvider, tracerProvider, }); @@ -2536,6 +2565,21 @@ test("handleInbox() records unverified HTTP signature details", async () => { "keyFetchError", ); assertEquals(event.attributes["http_signatures.key_fetch_status"], 410); + + const failures = recorder.getMeasurements( + "activitypub.signature.verification_failure", + ); + assertEquals(failures.length, 1); + assertEquals(failures[0].type, "counter"); + assertEquals(failures[0].value, 1); + assertEquals( + failures[0].attributes["activitypub.remote.host"], + "gone.example", + ); + assertEquals( + failures[0].attributes["activitypub.verification.failure_reason"], + "keyFetchError", + ); }); test("handleInbox() challenge policy enabled + unsigned request", async () => { diff --git a/packages/fedify/src/federation/handler.ts b/packages/fedify/src/federation/handler.ts index e407c18f9..f91846f35 100644 --- a/packages/fedify/src/federation/handler.ts +++ b/packages/fedify/src/federation/handler.ts @@ -15,6 +15,7 @@ import { import type { DocumentLoader } from "@fedify/vocab-runtime"; import { getLogger } from "@logtape/logtape"; import type { + MeterProvider, Span, SpanOptions, Tracer, @@ -63,6 +64,7 @@ import type { import { routeActivity } from "./inbox.ts"; import { KvKeyCache } from "./keycache.ts"; import type { KvKey, KvStore } from "./kv.ts"; +import { getFederationMetrics } from "./metrics.ts"; import type { MessageQueue } from "./mq.ts"; import { acceptsJsonLd } from "./negotiation.ts"; @@ -769,6 +771,11 @@ export interface InboxHandlerParameters { idempotencyStrategy?: | IdempotencyStrategy | IdempotencyKeyCallback; + /** + * The meter provider for recording metrics. + * @since 2.3.0 + */ + meterProvider?: MeterProvider; tracerProvider?: TracerProvider; } @@ -988,6 +995,11 @@ async function handleInboxInternal( }); if (verification.verified === false) { const reason = verification.reason; + const remoteHost = "keyId" in reason + ? reason.keyId?.hostname + : undefined; + getFederationMetrics(parameters.meterProvider) + .recordSignatureVerificationFailure(reason.type, remoteHost); logger.error( "Failed to verify the request's HTTP Signatures.", { @@ -1173,6 +1185,7 @@ async function handleInboxInternal( kvPrefixes, queue, span, + meterProvider: parameters.meterProvider, tracerProvider, idempotencyStrategy: parameters.idempotencyStrategy, }); diff --git a/packages/fedify/src/federation/inbox.ts b/packages/fedify/src/federation/inbox.ts index 3b242191c..db66692a8 100644 --- a/packages/fedify/src/federation/inbox.ts +++ b/packages/fedify/src/federation/inbox.ts @@ -3,6 +3,7 @@ import type { Activity } from "@fedify/vocab"; import { getLogger } from "@logtape/logtape"; import { context, + type MeterProvider, propagation, type Span, SpanKind, @@ -21,6 +22,7 @@ import type { KvKey, KvStore } from "./kv.ts"; import type { MessageQueue } from "./mq.ts"; import type { InboxMessage } from "./queue.ts"; import type { ActivityListenerSet } from "./activity-listener.ts"; +import { getDurationMs, getFederationMetrics } from "./metrics.ts"; export interface RouteActivityParameters { context: Context; @@ -39,6 +41,11 @@ export interface RouteActivityParameters { kvPrefixes: { activityIdempotence: KvKey }; queue?: MessageQueue; span: Span; + /** + * The meter provider for recording metrics. + * @since 2.3.0 + */ + meterProvider?: MeterProvider; tracerProvider?: TracerProvider; idempotencyStrategy?: | IdempotencyStrategy @@ -66,6 +73,7 @@ export async function routeActivity( kvPrefixes, queue, span, + meterProvider, tracerProvider, idempotencyStrategy, }: RouteActivityParameters, @@ -198,15 +206,24 @@ export async function routeActivity( const { class: cls, listener } = dispatched; span.updateName(`activitypub.dispatch_inbox_listener ${cls.name}`); try { - await listener( - inboxContextFactory( - recipient, - json, - activity?.id?.href, - getTypeId(activity!).href, - ), - activity!, - ); + const activityType = getTypeId(activity!).href; + const started = performance.now(); + try { + await listener( + inboxContextFactory( + recipient, + json, + activity?.id?.href, + activityType, + ), + activity!, + ); + } finally { + getFederationMetrics(meterProvider).recordInboxProcessingDuration( + activityType, + getDurationMs(started), + ); + } } catch (error) { try { await inboxErrorHandler?.(ctx, error as Error); diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts new file mode 100644 index 000000000..cdf69ea84 --- /dev/null +++ b/packages/fedify/src/federation/metrics.ts @@ -0,0 +1,133 @@ +import { + type Attributes, + type Counter, + type Histogram, + type MeterProvider, + metrics, +} from "@opentelemetry/api"; +import metadata from "../../deno.json" with { type: "json" }; + +class FederationMetrics { + readonly deliverySent: Counter; + readonly deliveryPermanentFailure: Counter; + readonly signatureVerificationFailure: Counter; + readonly deliveryDuration: Histogram; + readonly inboxProcessingDuration: Histogram; + + constructor(meterProvider: MeterProvider) { + const meter = meterProvider.getMeter(metadata.name, metadata.version); + this.deliverySent = meter.createCounter("activitypub.delivery.sent", { + description: "ActivityPub delivery attempts.", + unit: "{attempt}", + }); + this.deliveryPermanentFailure = meter.createCounter( + "activitypub.delivery.permanent_failure", + { + description: "ActivityPub deliveries abandoned as permanent failures.", + unit: "{failure}", + }, + ); + this.signatureVerificationFailure = meter.createCounter( + "activitypub.signature.verification_failure", + { + description: "ActivityPub HTTP Signature verification failures.", + unit: "{failure}", + }, + ); + this.deliveryDuration = meter.createHistogram( + "activitypub.delivery.duration", + { + description: "Duration of ActivityPub delivery attempts.", + unit: "ms", + }, + ); + this.inboxProcessingDuration = meter.createHistogram( + "activitypub.inbox.processing_duration", + { + description: "Duration of ActivityPub inbox listener processing.", + unit: "ms", + }, + ); + } + + recordDelivery( + inbox: URL, + durationMs: number, + success: boolean, + activityType?: string, + ): void { + const deliveryAttributes: Attributes = { + "activitypub.remote.host": getRemoteHost(inbox), + "activitypub.delivery.success": success, + }; + if (activityType != null) { + deliveryAttributes["activitypub.activity.type"] = activityType; + } + this.deliverySent.add(1, deliveryAttributes); + this.deliveryDuration.record(durationMs, { + "activitypub.remote.host": getRemoteHost(inbox), + }); + } + + recordPermanentFailure(inbox: URL, statusCode: number): void { + this.deliveryPermanentFailure.add(1, { + "activitypub.remote.host": getRemoteHost(inbox), + "http.response.status_code": statusCode, + }); + } + + recordSignatureVerificationFailure( + reason: string, + remoteHost?: string, + ): void { + const attributes: Attributes = { + "activitypub.verification.failure_reason": reason, + }; + if (remoteHost != null) { + attributes["activitypub.remote.host"] = remoteHost; + } + this.signatureVerificationFailure.add(1, attributes); + } + + recordInboxProcessingDuration( + activityType: string, + durationMs: number, + ): void { + this.inboxProcessingDuration.record(durationMs, { + "activitypub.activity.type": activityType, + }); + } +} + +const federationMetrics = new WeakMap(); + +/** + * Gets the cached Fedify metric instruments for a meter provider. + * @since 2.3.0 + */ +export function getFederationMetrics( + meterProvider: MeterProvider = metrics.getMeterProvider(), +): FederationMetrics { + let instruments = federationMetrics.get(meterProvider); + if (instruments == null) { + instruments = new FederationMetrics(meterProvider); + federationMetrics.set(meterProvider, instruments); + } + return instruments; +} + +/** + * Gets the bounded remote host attribute value for a URL. + * @since 2.3.0 + */ +export function getRemoteHost(url: URL): string { + return url.hostname; +} + +/** + * Gets an elapsed duration in milliseconds from a `performance.now()` value. + * @since 2.3.0 + */ +export function getDurationMs(start: number): number { + return Math.max(0, performance.now() - start); +} diff --git a/packages/fedify/src/federation/middleware.test.ts b/packages/fedify/src/federation/middleware.test.ts index 9390a1bf8..a1776349d 100644 --- a/packages/fedify/src/federation/middleware.test.ts +++ b/packages/fedify/src/federation/middleware.test.ts @@ -1,4 +1,5 @@ import { + createTestMeterProvider, createTestTracerProvider, mockDocumentLoader, test, @@ -3274,6 +3275,62 @@ test("FederationImpl.processQueuedTask()", async (t) => { await federation.processQueuedTask(undefined, inboxMessage); assertEquals(queuedMessages, [{ ...inboxMessage, attempt: 1 }]); }); + + await t.step("records queued inbox processing duration", async () => { + const kv = new MemoryKvStore(); + const [meterProvider, recorder] = createTestMeterProvider(); + const queue: MessageQueue = { + enqueue(_message, _options) { + return Promise.resolve(); + }, + listen(_handler, _options) { + return Promise.resolve(); + }, + }; + const federation = new FederationImpl({ + kv, + meterProvider, + queue, + }); + let handled = false; + federation.setInboxListeners("/users/{identifier}/inbox", "/inbox") + .on(vocab.Create, () => { + handled = true; + }); + + await federation.processQueuedTask( + undefined, + { + type: "inbox", + id: crypto.randomUUID(), + baseUrl: "https://example.com", + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + id: "https://remote.example/activities/1", + actor: "https://remote.example/users/alice", + object: { + type: "Note", + content: "Hello world", + }, + }, + started: new Date().toISOString(), + attempt: 0, + identifier: null, + traceContext: {}, + } satisfies InboxMessage, + ); + + assert(handled); + const durations = recorder.getMeasurements( + "activitypub.inbox.processing_duration", + ); + assertEquals(durations.length, 1); + assertEquals( + durations[0].attributes["activitypub.activity.type"], + "https://www.w3.org/ns/activitystreams#Create", + ); + }); }); test("FederationImpl.processQueuedTask() permanent failure", async (t) => { @@ -3305,6 +3362,12 @@ test("FederationImpl.processQueuedTask() permanent failure", async (t) => { options: { permanentFailureStatusCodes?: readonly number[]; nativeRetrial?: boolean; + meterProvider?: ConstructorParameters>[0][ + "meterProvider" + ]; + tracerProvider?: ConstructorParameters>[0][ + "tracerProvider" + ]; } = {}, ): PermanentFailureSetup { const kv = new MemoryKvStore(); @@ -3325,6 +3388,12 @@ test("FederationImpl.processQueuedTask() permanent failure", async (t) => { ...(options.permanentFailureStatusCodes ? { permanentFailureStatusCodes: options.permanentFailureStatusCodes } : {}), + ...(options.meterProvider + ? { meterProvider: options.meterProvider } + : {}), + ...(options.tracerProvider + ? { tracerProvider: options.tracerProvider } + : {}), }); federation.setInboxListeners("/users/{identifier}/inbox", "/inbox"); return { federation, queuedMessages }; @@ -3359,7 +3428,12 @@ test("FederationImpl.processQueuedTask() permanent failure", async (t) => { } await t.step("410 Gone triggers permanent failure handler", async () => { - const { federation, queuedMessages } = setup(); + const [meterProvider, recorder] = createTestMeterProvider(); + const [tracerProvider, exporter] = createTestTracerProvider(); + const { federation, queuedMessages } = setup({ + meterProvider, + tracerProvider, + }); let handlerCalled = false; let handlerValues: Record = {}; federation.setOutboxPermanentFailureHandler((_ctx, values) => { @@ -3391,6 +3465,36 @@ test("FederationImpl.processQueuedTask() permanent failure", async (t) => { ]); // Should NOT be re-enqueued for retry assertEquals(queuedMessages, []); + + const failures = recorder.getMeasurements( + "activitypub.delivery.permanent_failure", + ); + assertEquals(failures.length, 1); + assertEquals(failures[0].value, 1); + assertEquals( + failures[0].attributes["activitypub.remote.host"], + "gone.example", + ); + assertEquals( + failures[0].attributes["http.response.status_code"], + 410, + ); + + const events = exporter.getEvents( + "activitypub.outbox", + "activitypub.delivery.failed", + ); + assertEquals(events.length, 1); + assertEquals( + events[0].attributes?.["activitypub.remote.host"], + "gone.example", + ); + assertEquals(events[0].attributes?.["activitypub.delivery.attempt"], 0); + assertEquals( + events[0].attributes?.["activitypub.delivery.permanent_failure"], + true, + ); + assertEquals(events[0].attributes?.["http.response.status_code"], 410); }); await t.step("404 Not Found triggers permanent failure handler", async () => { diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 27b8a6dcc..67627ab03 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -32,6 +32,8 @@ import { lookupWebFinger } from "@fedify/webfinger"; import { getLogger, withContext } from "@logtape/logtape"; import { context, + type MeterProvider, + metrics, propagation, type Span, SpanKind, @@ -101,6 +103,7 @@ import { import { routeActivity } from "./inbox.ts"; import { KvKeyCache } from "./keycache.ts"; import type { KvKey, KvStore } from "./kv.ts"; +import { getFederationMetrics, getRemoteHost } from "./metrics.ts"; import type { MessageQueue } from "./mq.ts"; import { acceptsJsonLd } from "./negotiation.ts"; import type { @@ -248,6 +251,7 @@ export class FederationImpl inboxRetryPolicy: RetryPolicy; activityTransformers: readonly ActivityTransformer[]; _tracerProvider: TracerProvider | undefined; + _meterProvider: MeterProvider | undefined; firstKnock?: HttpMessageSignaturesSpec; inboxChallengePolicy?: InboxChallengePolicy; @@ -395,6 +399,7 @@ export class FederationImpl this.activityTransformers = options.activityTransformers ?? getDefaultActivityTransformers(); this._tracerProvider = options.tracerProvider; + this._meterProvider = options.meterProvider; this.firstKnock = options.firstKnock; } @@ -402,6 +407,10 @@ export class FederationImpl return this._tracerProvider ?? trace.getTracerProvider(); } + get meterProvider(): MeterProvider { + return this._meterProvider ?? metrics.getMeterProvider(); + } + _initializeRouter(): void { this.router.add("/.well-known/webfinger", "webfinger"); this.router.add("/.well-known/nodeinfo", "nodeInfoJrd"); @@ -673,10 +682,21 @@ export class FederationImpl this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, ), + meterProvider: this.meterProvider, tracerProvider: this.tracerProvider, }); } catch (error) { span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); + span.addEvent("activitypub.delivery.failed", { + "activitypub.remote.host": getRemoteHost(new URL(message.inbox)), + "activitypub.delivery.attempt": message.attempt, + "activitypub.delivery.permanent_failure": + error instanceof SendActivityError && + this.permanentFailureStatusCodes.includes(error.statusCode), + ...(error instanceof SendActivityError + ? { "http.response.status_code": error.statusCode } + : {}), + }); const loaderOptions = this.#getLoaderOptions(message.baseUrl); const activity = await Activity.fromJsonLd(message.activity, { contextLoader: this.contextLoaderFactory(loaderOptions), @@ -699,6 +719,10 @@ export class FederationImpl error instanceof SendActivityError && this.permanentFailureStatusCodes.includes(error.statusCode) ) { + getFederationMetrics(this.meterProvider).recordPermanentFailure( + error.inbox, + error.statusCode, + ); logger.warn( "Permanent delivery failure for activity {activityId} to " + "{inbox} ({status}); not retrying.", @@ -861,15 +885,25 @@ export class FederationImpl const { class: cls, listener } = dispatched; span.updateName(`activitypub.dispatch_inbox_listener ${cls.name}`); try { - await listener( - context.toInboxContext( - message.identifier, - message.activity, - activity.id?.href, - getTypeId(activity).href, - ), - activity, - ); + const activityType = getTypeId(activity).href; + const started = performance.now(); + try { + await listener( + context.toInboxContext( + message.identifier, + message.activity, + activity.id?.href, + activityType, + ), + activity, + ); + } finally { + getFederationMetrics(this.meterProvider) + .recordInboxProcessingDuration( + activityType, + Math.max(0, performance.now() - started), + ); + } } catch (error) { try { await this.inboxErrorHandler?.(context, error as Error); @@ -1194,6 +1228,7 @@ export class FederationImpl this.kvPrefixes.httpMessageSignaturesSpec, this.firstKnock, ), + meterProvider: this.meterProvider, tracerProvider: this.tracerProvider, }), ); @@ -1550,6 +1585,7 @@ export class FederationImpl signatureTimeWindow: this.signatureTimeWindow, skipSignatureVerification: this.skipSignatureVerification, inboxChallengePolicy: this.inboxChallengePolicy, + meterProvider: this.meterProvider, tracerProvider: this.tracerProvider, idempotencyStrategy: this.idempotencyStrategy, }); @@ -1785,6 +1821,10 @@ export class ContextImpl implements Context { return this.federation.tracerProvider; } + get meterProvider(): MeterProvider { + return this.federation.meterProvider; + } + getNodeInfoUri(): URL { const path = this.federation.router.build("nodeInfo", {}); if (path == null) { @@ -3062,6 +3102,7 @@ async function forwardActivityInternal( activityType: ctx.activityType, inbox: new URL(inbox), sharedInbox: inboxes[inbox].sharedInbox, + meterProvider: ctx.meterProvider, tracerProvider: ctx.tracerProvider, specDeterminer: new KvSpecDeterminer( ctx.federation.kv, diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index 55fc4d039..ee508f373 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -1,4 +1,5 @@ import { + createTestMeterProvider, createTestTracerProvider, mockDocumentLoader, test, @@ -16,10 +17,17 @@ import { assert, assertEquals, assertFalse, + assertGreaterOrEqual, assertInstanceOf, assertNotEquals, assertRejects, } from "@std/assert"; +import { + AggregationTemporality, + InMemoryMetricExporter, + MeterProvider, + PeriodicExportingMetricReader, +} from "@opentelemetry/sdk-metrics"; import fetchMock from "fetch-mock"; import { verifyRequest } from "../sig/http.ts"; import { doesActorOwnKey } from "../sig/owner.ts"; @@ -503,3 +511,147 @@ test("sendActivity() records OpenTelemetry span events", async (t) => { fetchMock.hardReset(); }); }); + +test("sendActivity() records OpenTelemetry delivery metrics", async (t) => { + const [meterProvider, recorder] = createTestMeterProvider(); + fetchMock.spyGlobal(); + + await t.step("successful send", async () => { + fetchMock.post("https://metrics.example:8443/inbox/path?x=1", { + status: 202, + }); + + const activity = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + id: "https://example.com/activity", + actor: "https://example.com/person", + }; + + await sendActivity({ + activity, + activityId: "https://example.com/activity", + activityType: "https://www.w3.org/ns/activitystreams#Create", + keys: [], + inbox: new URL("https://metrics.example:8443/inbox/path?x=1"), + meterProvider, + }); + + const sent = recorder.getMeasurements("activitypub.delivery.sent"); + assertEquals(sent.length, 1); + assertEquals(sent[0].type, "counter"); + assertEquals(sent[0].value, 1); + assertEquals( + sent[0].attributes["activitypub.remote.host"], + "metrics.example", + ); + assertEquals( + sent[0].attributes["activitypub.activity.type"], + "https://www.w3.org/ns/activitystreams#Create", + ); + assertEquals(sent[0].attributes["activitypub.delivery.success"], true); + + const durations = recorder.getMeasurements( + "activitypub.delivery.duration", + ); + assertEquals(durations.length, 1); + assertEquals(durations[0].type, "histogram"); + assertGreaterOrEqual(durations[0].value, 0); + assertEquals( + durations[0].attributes["activitypub.remote.host"], + "metrics.example", + ); + + recorder.clear(); + fetchMock.hardReset(); + }); + + await t.step("failed HTTP response", async () => { + fetchMock.spyGlobal(); + fetchMock.post("https://metrics.example/inbox", { + status: 500, + body: "failed", + }); + + const activity = { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Follow", + id: "https://example.com/follow", + actor: "https://example.com/person", + }; + + await assertRejects( + () => + sendActivity({ + activity, + activityId: "https://example.com/follow", + activityType: "https://www.w3.org/ns/activitystreams#Follow", + keys: [], + inbox: new URL("https://metrics.example/inbox"), + meterProvider, + }), + SendActivityError, + ); + + const sent = recorder.getMeasurements("activitypub.delivery.sent"); + assertEquals(sent.length, 1); + assertEquals(sent[0].attributes["activitypub.delivery.success"], false); + assertEquals( + sent[0].attributes["activitypub.activity.type"], + "https://www.w3.org/ns/activitystreams#Follow", + ); + + const durations = recorder.getMeasurements( + "activitypub.delivery.duration", + ); + assertEquals(durations.length, 1); + assertGreaterOrEqual(durations[0].value, 0); + + recorder.clear(); + fetchMock.hardReset(); + }); +}); + +test("sendActivity() exports delivery metrics through OpenTelemetry SDK", async () => { + const exporter = new InMemoryMetricExporter( + AggregationTemporality.CUMULATIVE, + ); + const reader = new PeriodicExportingMetricReader({ + exporter, + exportIntervalMillis: 60_000, + }); + const meterProvider = new MeterProvider({ readers: [reader] }); + fetchMock.spyGlobal(); + fetchMock.post("https://sdk-metrics.example/inbox", { status: 202 }); + + await sendActivity({ + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + id: "https://example.com/activity", + actor: "https://example.com/person", + }, + activityId: "https://example.com/activity", + activityType: "https://www.w3.org/ns/activitystreams#Create", + keys: [], + inbox: new URL("https://sdk-metrics.example/inbox"), + meterProvider, + }); + + await meterProvider.forceFlush(); + const exportedMetrics = exporter.getMetrics() + .flatMap((resourceMetrics) => resourceMetrics.scopeMetrics) + .flatMap((scopeMetrics) => scopeMetrics.metrics); + const sent = exportedMetrics.find((metric) => + metric.descriptor.name === "activitypub.delivery.sent" + ); + assert(sent != null); + assertEquals(sent.dataPoints.length, 1); + assertEquals( + sent.dataPoints[0].attributes["activitypub.remote.host"], + "sdk-metrics.example", + ); + + await meterProvider.shutdown(); + fetchMock.hardReset(); +}); diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 4389a7a54..006212a8f 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -1,6 +1,7 @@ import type { Recipient } from "@fedify/vocab"; import { getLogger } from "@logtape/logtape"; import { + type MeterProvider, type Span, SpanKind, SpanStatusCode, @@ -12,6 +13,7 @@ import { doubleKnock, type HttpMessageSignaturesSpecDeterminer, } from "../sig/http.ts"; +import { getDurationMs, getFederationMetrics } from "./metrics.ts"; /** * Parameters for {@link extractInboxes}. @@ -140,6 +142,13 @@ export interface SendActivityParameters { */ readonly specDeterminer?: HttpMessageSignaturesSpecDeterminer; + /** + * The meter provider for recording metrics. + * If omitted, the global meter provider is used. + * @since 2.3.0 + */ + readonly meterProvider?: MeterProvider; + /** * The tracer provider for tracing the request. * If omitted, the global tracer provider is used. @@ -232,15 +241,20 @@ async function sendActivityInternal( { activity, activityId, + activityType, keys, inbox, headers, specDeterminer, + meterProvider, tracerProvider, }: SendActivityParameters, span: Span, ): Promise { const logger = getLogger(["fedify", "federation", "outbox"]); + const federationMetrics = getFederationMetrics(meterProvider); + const started = performance.now(); + let deliverySuccess = false; headers = new Headers(headers); headers.set("Content-Type", "application/activity+json"); const request = new Request(inbox, { @@ -284,44 +298,61 @@ async function sendActivityInternal( error, }, ); + federationMetrics.recordDelivery( + inbox, + getDurationMs(started), + false, + activityType, + ); throw error; } - if (!response.ok) { - let error: string; - try { - error = await readLimitedResponseBody( - response, - MAX_ERROR_RESPONSE_BODY_BYTES, + try { + if (!response.ok) { + let error: string; + try { + error = await readLimitedResponseBody( + response, + MAX_ERROR_RESPONSE_BODY_BYTES, + ); + } catch (_) { + error = ""; + } + logger.error( + "Failed to send activity {activityId} to {inbox} ({status} " + + "{statusText}):\n{error}", + { + activityId, + inbox: inbox.href, + status: response.status, + statusText: response.statusText, + error, + }, ); - } catch (_) { - error = ""; - } - logger.error( - "Failed to send activity {activityId} to {inbox} ({status} " + - "{statusText}):\n{error}", - { - activityId, - inbox: inbox.href, - status: response.status, - statusText: response.statusText, + throw new SendActivityError( + inbox, + response.status, + `Failed to send activity ${activityId} to ${inbox.href} ` + + `(${response.status} ${response.statusText}):\n${error}`, error, - }, - ); - throw new SendActivityError( + ); + } + + deliverySuccess = true; + + // Record the sent activity with delivery details + span.addEvent("activitypub.activity.sent", { + "activitypub.activity.json": JSON.stringify(activity), + "activitypub.inbox.url": inbox.href, + "activitypub.activity.id": activityId ?? "", + }); + } finally { + federationMetrics.recordDelivery( inbox, - response.status, - `Failed to send activity ${activityId} to ${inbox.href} ` + - `(${response.status} ${response.statusText}):\n${error}`, - error, + getDurationMs(started), + deliverySuccess, + activityType, ); } - - // Record the sent activity with delivery details - span.addEvent("activitypub.activity.sent", { - "activitypub.activity.json": JSON.stringify(activity), - "activitypub.inbox.url": inbox.href, - "activitypub.activity.id": activityId ?? "", - }); } /** diff --git a/packages/fedify/src/testing/context.ts b/packages/fedify/src/testing/context.ts index e84dee5fa..8787d11ce 100644 --- a/packages/fedify/src/testing/context.ts +++ b/packages/fedify/src/testing/context.ts @@ -4,7 +4,7 @@ import { traverseCollection as globalTraverseCollection, } from "@fedify/vocab"; import { lookupWebFinger as globalLookupWebFinger } from "@fedify/webfinger"; -import { trace } from "@opentelemetry/api"; +import { metrics, trace } from "@opentelemetry/api"; import type { Context, InboxContext, @@ -28,6 +28,7 @@ export function createContext( data, documentLoader, contextLoader, + meterProvider, tracerProvider, clone, getNodeInfoUri, @@ -63,6 +64,7 @@ export function createContext( hostname: url.hostname, documentLoader: documentLoader ?? mockDocumentLoader, contextLoader: contextLoader ?? mockDocumentLoader, + meterProvider: meterProvider ?? metrics.getMeterProvider(), tracerProvider: tracerProvider ?? trace.getTracerProvider(), clone: clone ?? ((data) => createContext({ ...values, data })), getNodeInfoUri: getNodeInfoUri ?? throwRouterError, diff --git a/packages/fixture/src/otel.ts b/packages/fixture/src/otel.ts index f6cb48a04..680730bff 100644 --- a/packages/fixture/src/otel.ts +++ b/packages/fixture/src/otel.ts @@ -4,6 +4,25 @@ import { SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-base"; import { ExportResultCode } from "@opentelemetry/core"; +import type { + Attributes, + BatchObservableCallback, + Context, + Counter, + Gauge, + Histogram, + Meter, + MeterOptions, + MeterProvider, + MetricAttributes, + MetricOptions, + Observable, + ObservableCallback, + ObservableCounter, + ObservableGauge, + ObservableUpDownCounter, + UpDownCounter, +} from "@opentelemetry/api"; /** * A test spy for OpenTelemetry spans that captures all spans and events. @@ -76,3 +95,238 @@ export function createTestTracerProvider(): [ }); return [provider, exporter]; } + +/** + * A metric measurement captured by {@link TestMetricRecorder}. + * @since 2.3.0 + */ +export interface TestMetricMeasurement { + /** + * The metric instrument name. + * @since 2.3.0 + */ + readonly name: string; + + /** + * The instrument type that recorded the measurement. + * @since 2.3.0 + */ + readonly type: "counter" | "histogram" | "gauge" | "upDownCounter"; + + /** + * The recorded metric value. + * @since 2.3.0 + */ + readonly value: number; + + /** + * The attributes recorded with the measurement. + * @since 2.3.0 + */ + readonly attributes: Attributes; +} + +/** + * A test recorder for OpenTelemetry metric measurements. + * @since 2.3.0 + */ +export class TestMetricRecorder { + /** + * The captured metric measurements. + * @since 2.3.0 + */ + public measurements: TestMetricMeasurement[] = []; + + /** + * Records a metric measurement. + * @since 2.3.0 + */ + record(measurement: TestMetricMeasurement): void { + this.measurements.push(measurement); + } + + /** + * Gets all measurements with the given metric name. + * @since 2.3.0 + */ + getMeasurements(name: string): TestMetricMeasurement[] { + return this.measurements.filter((measurement) => measurement.name === name); + } + + /** + * Gets the first measurement with the given metric name. + * @since 2.3.0 + */ + getMeasurement(name: string): TestMetricMeasurement | undefined { + return this.measurements.find((measurement) => measurement.name === name); + } + + /** + * Clears all captured measurements. + * @since 2.3.0 + */ + clear(): void { + this.measurements = []; + } +} + +class TestCounter + implements Counter { + constructor( + private readonly name: string, + private readonly recorder: TestMetricRecorder, + private readonly type: TestMetricMeasurement["type"] = "counter", + ) { + } + + add(value: number, attributes?: AttributesTypes, _context?: Context): void { + this.recorder.record({ + name: this.name, + type: this.type, + value, + attributes: { ...(attributes ?? {}) }, + }); + } +} + +class TestHistogram< + AttributesTypes extends MetricAttributes = MetricAttributes, +> implements Histogram, Gauge { + constructor( + private readonly name: string, + private readonly recorder: TestMetricRecorder, + private readonly type: TestMetricMeasurement["type"] = "histogram", + ) { + } + + record( + value: number, + attributes?: AttributesTypes, + _context?: Context, + ): void { + this.recorder.record({ + name: this.name, + type: this.type, + value, + attributes: { ...(attributes ?? {}) }, + }); + } +} + +class TestObservable< + AttributesTypes extends MetricAttributes = MetricAttributes, +> implements Observable { + readonly callbacks = new Set>(); + + addCallback(callback: ObservableCallback): void { + this.callbacks.add(callback); + } + + removeCallback(callback: ObservableCallback): void { + this.callbacks.delete(callback); + } +} + +class TestMeter implements Meter { + constructor(private readonly recorder: TestMetricRecorder) { + } + + createCounter( + name: string, + _options?: MetricOptions, + ): Counter { + return new TestCounter(name, this.recorder); + } + + createUpDownCounter< + AttributesTypes extends MetricAttributes = MetricAttributes, + >( + name: string, + _options?: MetricOptions, + ): UpDownCounter { + return new TestCounter(name, this.recorder, "upDownCounter"); + } + + createHistogram( + name: string, + _options?: MetricOptions, + ): Histogram { + return new TestHistogram(name, this.recorder); + } + + createGauge( + name: string, + _options?: MetricOptions, + ): Gauge { + return new TestHistogram(name, this.recorder, "gauge"); + } + + createObservableCounter< + AttributesTypes extends MetricAttributes = MetricAttributes, + >( + _name: string, + _options?: MetricOptions, + ): ObservableCounter { + return new TestObservable(); + } + + createObservableUpDownCounter< + AttributesTypes extends MetricAttributes = MetricAttributes, + >( + _name: string, + _options?: MetricOptions, + ): ObservableUpDownCounter { + return new TestObservable(); + } + + createObservableGauge< + AttributesTypes extends MetricAttributes = MetricAttributes, + >( + _name: string, + _options?: MetricOptions, + ): ObservableGauge { + return new TestObservable(); + } + + addBatchObservableCallback< + AttributesTypes extends MetricAttributes = MetricAttributes, + >( + _callback: BatchObservableCallback, + _observables: Observable[], + ): void { + } + + removeBatchObservableCallback< + AttributesTypes extends MetricAttributes = MetricAttributes, + >( + _callback: BatchObservableCallback, + _observables: Observable[], + ): void { + } +} + +class TestMeterProvider implements MeterProvider { + constructor(private readonly recorder: TestMetricRecorder) { + } + + getMeter( + _name: string, + _version?: string, + _options?: MeterOptions, + ): Meter { + return new TestMeter(this.recorder); + } +} + +/** + * Creates a test meter provider with a test recorder. + * @returns A tuple of [meterProvider, testRecorder]. + * @since 2.3.0 + */ +export function createTestMeterProvider(): [ + MeterProvider, + TestMetricRecorder, +] { + const recorder = new TestMetricRecorder(); + return [new TestMeterProvider(recorder), recorder]; +} diff --git a/packages/testing/src/context.ts b/packages/testing/src/context.ts index 7721577fe..a4771cf14 100644 --- a/packages/testing/src/context.ts +++ b/packages/testing/src/context.ts @@ -37,6 +37,29 @@ const noopTracerProvider: any = { }), }; +const noopMeterProvider: any = { + getMeter: () => ({ + createCounter: () => ({ add: () => undefined }), + createGauge: () => ({ record: () => undefined }), + createHistogram: () => ({ record: () => undefined }), + createObservableCounter: () => ({ + addCallback: () => undefined, + removeCallback: () => undefined, + }), + createObservableGauge: () => ({ + addCallback: () => undefined, + removeCallback: () => undefined, + }), + createObservableUpDownCounter: () => ({ + addCallback: () => undefined, + removeCallback: () => undefined, + }), + createUpDownCounter: () => ({ add: () => undefined }), + addBatchObservableCallback: () => undefined, + removeBatchObservableCallback: () => undefined, + }), +}; + // NOTE: Copied from @fedify/fedify/testing/context.ts // Not exported - used internally only. Public API is in mock.ts @@ -54,6 +77,7 @@ function createContext( data, documentLoader, contextLoader, + meterProvider, tracerProvider, clone, getNodeInfoUri, @@ -89,6 +113,7 @@ function createContext( hostname: url.hostname, documentLoader: documentLoader ?? mockDocumentLoader, contextLoader: contextLoader ?? mockDocumentLoader, + meterProvider: meterProvider ?? noopMeterProvider, tracerProvider: tracerProvider ?? noopTracerProvider, clone: clone ?? ((data) => createContext({ ...values, data })), getNodeInfoUri: getNodeInfoUri ?? throwRouterError, diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index c618cd0a2..12b96741c 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -56,6 +56,29 @@ const noopTracerProvider: any = { }), }; +const noopMeterProvider: any = { + getMeter: () => ({ + createCounter: () => ({ add: () => undefined }), + createGauge: () => ({ record: () => undefined }), + createHistogram: () => ({ record: () => undefined }), + createObservableCounter: () => ({ + addCallback: () => undefined, + removeCallback: () => undefined, + }), + createObservableGauge: () => ({ + addCallback: () => undefined, + removeCallback: () => undefined, + }), + createObservableUpDownCounter: () => ({ + addCallback: () => undefined, + removeCallback: () => undefined, + }), + createUpDownCounter: () => ({ add: () => undefined }), + addBatchObservableCallback: () => undefined, + removeBatchObservableCallback: () => undefined, + }), +}; + /** * Helper function to expand URI templates used by the mock. * Supports the RFC 6570 operators accepted by Fedify's identifier paths. @@ -282,6 +305,7 @@ class MockFederation implements Federation { private options: { contextData?: TContextData; origin?: string; + meterProvider?: any; tracerProvider?: any; } = {}, ) { @@ -513,6 +537,8 @@ class MockFederation implements Federation { request, data: contextData, federation: mockFederation as any, + meterProvider: this.options.meterProvider, + tracerProvider: this.options.tracerProvider, }); } @@ -784,6 +810,7 @@ export function createFederation( options: { contextData?: TContextData; origin?: string; + meterProvider?: any; tracerProvider?: any; } = {}, ): TestFederation { @@ -842,6 +869,7 @@ class MockContext implements Context { readonly federation: Federation; readonly documentLoader: DocumentLoader; readonly contextLoader: DocumentLoader; + readonly meterProvider: any; readonly tracerProvider: any; readonly request: Request; readonly url: URL; @@ -861,6 +889,7 @@ class MockContext implements Context { federation: Federation; documentLoader?: DocumentLoader; contextLoader?: DocumentLoader; + meterProvider?: any; tracerProvider?: any; }, ) { @@ -880,6 +909,7 @@ class MockContext implements Context { documentUrl: url, })); this.contextLoader = options.contextLoader ?? this.documentLoader; + this.meterProvider = options.meterProvider ?? noopMeterProvider; this.tracerProvider = options.tracerProvider ?? noopTracerProvider; } @@ -928,6 +958,7 @@ class MockContext implements Context { federation: this.federation, documentLoader: this.documentLoader, contextLoader: this.contextLoader, + meterProvider: this.meterProvider, tracerProvider: this.tracerProvider, }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2da678335..b0253d9b6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,6 +45,9 @@ catalogs: '@opentelemetry/exporter-trace-otlp-proto': specifier: ^0.211.0 version: 0.211.0 + '@opentelemetry/sdk-metrics': + specifier: 2.5.0 + version: 2.5.0 '@opentelemetry/sdk-node': specifier: ^0.211.0 version: 0.211.0 @@ -1201,6 +1204,9 @@ importers: '@fedify/vocab-tools': specifier: workspace:^ version: link:../vocab-tools + '@opentelemetry/sdk-metrics': + specifier: 'catalog:' + version: 2.5.0(@opentelemetry/api@1.9.0) '@std/assert': specifier: jsr:^0.226.0 version: '@jsr/std__assert@0.226.0' diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 657564a6d..394778464 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -56,6 +56,7 @@ catalog: "@opentelemetry/context-async-hooks": ^2.5.0 "@opentelemetry/core": ^2.5.0 "@opentelemetry/sdk-node": ^0.211.0 + "@opentelemetry/sdk-metrics": 2.5.0 "@opentelemetry/sdk-trace-base": ^2.5.0 "@opentelemetry/semantic-conventions": ^1.39.0 "@optique/config": ^1.0.2 From 34189b81a244a232b1130111d69d245c8c6136b5 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 5 May 2026 20:19:46 +0900 Subject: [PATCH 02/16] Document federation metrics Describe the new meter provider option, emitted federation metrics, and the queued delivery failure span event. Update the changelog and overview copy so Fedify's OpenTelemetry support covers metrics as well as tracing. Refs: https://github.com/fedify-dev/fedify/issues/619 Assisted-by: gpt-5.5 --- CHANGES.md | 17 +++ docs/manual/federation.md | 12 +++ docs/manual/opentelemetry.md | 199 +++++++++++++++++++++++++---------- docs/why.md | 4 +- 4 files changed, 173 insertions(+), 59 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index c12781771..252bdd799 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,11 +22,28 @@ To be released. and delayed counts, and `ParallelMessageQueue` delegates depth reporting to its wrapped queue when supported. [[#735], [#748]] + - Added OpenTelemetry metrics for ActivityPub delivery attempts, permanent + delivery failures, inbox listener processing duration, and HTTP Signature + verification failures. Applications can pass the new `meterProvider` + option to `createFederation()`, and `Context.meterProvider` exposes the + provider available to request, inbox, and outbox code. [[#619]] + + - Added the `activitypub.delivery.failed` span event to queued outbox + delivery spans so retry and permanent-failure decisions include the + remote host, attempt number, and HTTP status code when available. [[#619]] + +[#619]: https://github.com/fedify-dev/fedify/issues/619 [#735]: https://github.com/fedify-dev/fedify/issues/735 [#748]: https://github.com/fedify-dev/fedify/pull/748 [#752]: https://github.com/fedify-dev/fedify/issues/752 [#753]: https://github.com/fedify-dev/fedify/pull/753 +### @fedify/fixture + + - Added `createTestMeterProvider()` and `TestMetricRecorder` helpers for + asserting OpenTelemetry metric measurements in runtime-agnostic tests. + [[#619]] + ### @fedify/amqp - Added `AmqpMessageQueue.getDepth()` for reporting queued, ready, and diff --git a/docs/manual/federation.md b/docs/manual/federation.md index 5e8b536df..3de062cfe 100644 --- a/docs/manual/federation.md +++ b/docs/manual/federation.md @@ -444,6 +444,18 @@ For more information, see the [*OpenTelemetry* section](./opentelemetry.md). [`trace.getTracerProvider()`]: https://open-telemetry.github.io/opentelemetry-js/classes/_opentelemetry_api._opentelemetry_api.TraceAPI.html#gettracerprovider +### `meterProvider` + +*This API is available since Fedify 2.3.0.* + +The OpenTelemetry meter provider that the `Federation` object uses to record +Fedify metrics. If omitted, it is configured to use the default meter provider +(i.e., [`metrics.getMeterProvider()`]). + +For more information, see the [*OpenTelemetry* section](./opentelemetry.md). + +[`metrics.getMeterProvider()`]: https://open-telemetry.github.io/opentelemetry-js/classes/_opentelemetry_api._opentelemetry_api.MetricsAPI.html#getmeterprovider + Builder pattern for structuring ------------------------------- diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index 39e4e684a..18ef2454f 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -2,7 +2,8 @@ description: >- OpenTelemetry is a set of APIs, libraries, agents, and instrumentation to provide observability to your applications. Fedify supports OpenTelemetry - for tracing. This document explains how to use OpenTelemetry with Fedify. + for tracing and metrics. This document explains how to use OpenTelemetry + with Fedify. --- OpenTelemetry @@ -12,8 +13,8 @@ OpenTelemetry [OpenTelemetry] is a standardized set of APIs, libraries, agents, and instrumentation to provide observability to your applications. Fedify supports -OpenTelemetry for tracing. This document explains how to use OpenTelemetry with -Fedify. +OpenTelemetry for tracing and metrics. This document explains how to use +OpenTelemetry with Fedify. [OpenTelemetry]: https://opentelemetry.io/ @@ -26,10 +27,11 @@ Setting up OpenTelemetry > support. See the [*Using Deno's built-in OpenTelemetry support* > section](#using-deno-s-built-in-opentelemetry-support) for more details. -To trace your Fedify application with OpenTelemetry, you need to set up the -OpenTelemetry SDK. First of all, you need to install the OpenTelemetry SDK and -the tracer exporter you want to use. For example, if you want to use the trace -exporter for OTLP (http/protobuf), you should install the following packages: +To trace your Fedify application and collect metrics with OpenTelemetry, you +need to set up the OpenTelemetry SDK. First of all, you need to install the +OpenTelemetry SDK and the exporter you want to use. For example, if you want +to use the trace exporter for OTLP (http/protobuf), you should install the +following packages: ::: code-group @@ -146,6 +148,34 @@ const federation = createFederation({ [*OpenTelemetry Support* section]: https://docs.sentry.io/platforms/javascript/guides/node/opentelemetry/ +Explicit [`MeterProvider`] configuration +---------------------------------------- + +*This API is available since Fedify 2.3.0.* + +The `createFederation()` function also accepts the +[`meterProvider`](./federation.md#meterprovider) option to explicitly configure +the [`MeterProvider`] for OpenTelemetry metrics. If it is omitted, Fedify uses +the global default [`MeterProvider`] provided by the OpenTelemetry SDK. + +~~~~ typescript twoslash +import type { KvStore } from "@fedify/fedify"; +// ---cut-before--- +import { createFederation } from "@fedify/fedify"; +import { metrics } from "@opentelemetry/api"; + +const federation = createFederation({ +// ---cut-start--- + kv: null as unknown as KvStore, +// ---cut-end--- + // Omitted for brevity; see the related section for details. + meterProvider: metrics.getMeterProvider(), +}); +~~~~ + +[`MeterProvider`]: https://open-telemetry.github.io/opentelemetry-js/interfaces/_opentelemetry_api._opentelemetry_api.MeterProvider.html + + Instrumented spans ------------------ @@ -204,6 +234,7 @@ The following span events are recorded: | ------------------------------- | --------------------------- | -------------------------------------------------------------------------------- | | `activitypub.activity.received` | `activitypub.inbox` | Records full activity JSON and verification status when an activity is received. | | `activitypub.activity.sent` | `activitypub.send_activity` | Records full activity JSON and delivery details when an activity is sent. | +| `activitypub.delivery.failed` | `activitypub.outbox` | Records queued outbox delivery failure details before retry or abandonment. | | `activitypub.object.fetched` | `activitypub.lookup_object` | Records full object JSON when successfully fetched. | [span events]: https://opentelemetry.io/docs/concepts/signals/traces/#span-events @@ -236,12 +267,61 @@ Each span event includes attributes with detailed information: - `activitypub.inbox.url`: The inbox URL where the activity was delivered - `activitypub.activity.id`: The activity ID +**`activitypub.delivery.failed` event attributes:** + + - `activitypub.remote.host`: The remote inbox host + - `activitypub.delivery.attempt`: The zero-based queue delivery attempt + - `activitypub.delivery.permanent_failure`: Whether Fedify will abandon the + delivery instead of retrying + - `http.response.status_code` (optional): The HTTP response status code + returned by the remote inbox + **`activitypub.object.fetched` event attributes:** - `activitypub.object.type`: The type URI of the fetched object - `activitypub.object.json`: The complete object JSON +Instrumented metrics +-------------------- + +*This API is available since Fedify 2.3.0.* + +Fedify records the following OpenTelemetry metrics: + +| Metric name | Instrument | Unit | Description | +| -------------------------------------------- | ---------- | ----------- | ------------------------------------------------------------- | +| `activitypub.delivery.sent` | Counter | `{attempt}` | Counts outgoing ActivityPub delivery attempts. | +| `activitypub.delivery.permanent_failure` | Counter | `{failure}` | Counts outgoing deliveries abandoned as permanent failures. | +| `activitypub.delivery.duration` | Histogram | `ms` | Measures outgoing ActivityPub delivery attempt duration. | +| `activitypub.inbox.processing_duration` | Histogram | `ms` | Measures inbox listener processing duration. | +| `activitypub.signature.verification_failure` | Counter | `{failure}` | Counts failed HTTP Signature verification for inbox requests. | + +### Metric attributes + +`activitypub.delivery.sent` +: `activitypub.remote.host`, `activitypub.delivery.success`, and + `activitypub.activity.type` when Fedify knows the activity type. + +`activitypub.delivery.permanent_failure` +: `activitypub.remote.host` and `http.response.status_code`. + +`activitypub.delivery.duration` +: `activitypub.remote.host`. + +`activitypub.inbox.processing_duration` +: `activitypub.activity.type`. + +`activitypub.signature.verification_failure` +: `activitypub.verification.failure_reason`, plus + `activitypub.remote.host` when the failed signature includes a key ID. + +Fedify records `activitypub.remote.host` as the URL hostname only; ports, paths, +and query strings are deliberately excluded to keep metric cardinality bounded. +Activity types use the same qualified URI form as Fedify's trace attributes, +for example `https://www.w3.org/ns/activitystreams#Create`. + + Semantic [attributes] for ActivityPub ------------------------------------- @@ -250,56 +330,61 @@ for ActivityPub as of November 2024. However, Fedify provides a set of semantic [attributes] for ActivityPub. The following table shows the semantic attributes for ActivityPub: -| Attribute | Type | Description | Example | -| ------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | -| `activitypub.activity.id` | string | The URI of the activity object. | `"https://example.com/activity/1"` | -| `activitypub.activity.type` | string[] | The qualified URI(s) of the activity type(s). | `["https://www.w3.org/ns/activitystreams#Create"]` | -| `activitypub.activity.to` | string[] | The URI(s) of the recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | -| `activitypub.activity.cc` | string[] | The URI(s) of the carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | -| `activitypub.activity.bto` | string[] | The URI(s) of the blind recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | -| `activitypub.activity.bcc` | string[] | The URI(s) of the blind carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | -| `activitypub.activity.retries` | int | The ordinal number of activity resending attempt (if and only if it's retried). | `3` | -| `activitypub.actor.id` | string | The URI of the actor object. | `"https://example.com/actor/1"` | -| `activitypub.actor.key.cached` | boolean | Whether the actor's public keys are cached. | `true` | -| `activitypub.actor.type` | string[] | The qualified URI(s) of the actor type(s). | `["https://www.w3.org/ns/activitystreams#Person"]` | -| `activitypub.key.id` | string | The URI of the cryptographic key being verified. | `"https://example.com/actor/1#main-key"` | -| `activitypub.key_ownership.method` | string | The method used to verify key ownership (`owner_id` or `actor_fetch`). | `"actor_fetch"` | -| `activitypub.key_ownership.verified` | boolean | Whether the key ownership was successfully verified. | `true` | -| `activitypub.collection.id` | string | The URI of the collection object. | `"https://example.com/collection/1"` | -| `activitypub.collection.type` | string[] | The qualified URI(s) of the collection type(s). | `["https://www.w3.org/ns/activitystreams#OrderedCollection"]` | -| `activitypub.collection.total_items` | int | The total number of items in the collection. | `42` | -| `activitypub.object.id` | string | The URI of the object or the object enclosed by the activity. | `"https://example.com/object/1"` | -| `activitypub.object.type` | string[] | The qualified URI(s) of the object type(s). | `["https://www.w3.org/ns/activitystreams#Note"]` | -| `activitypub.object.in_reply_to` | string[] | The URI(s) of the original object to which the object reply. | `["https://example.com/object/1"]` | -| `activitypub.inboxes` | int | The number of inboxes the activity is sent to. | `12` | -| `activitypub.shared_inbox` | boolean | Whether the activity is sent to the shared inbox. | `true` | -| `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | -| `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | -| `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | -| `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | -| `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | -| `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | -| `fedify.collection.cursor` | string | The cursor of the collection. | `"eyJpZCI6IjEiLCJ0eXBlIjoiT3JkZXJlZENvbGxlY3Rpb24ifQ=="` | -| `fedify.collection.items` | number | The number of items in the collection page. It can be less than the total items. | `10` | -| `http.redirect.url` | string | The redirect URL when a document fetch results in a redirect. | `"https://example.com/new-location"` | -| `http.response.status_code` | int | The HTTP response status code. | `200` | -| `http_signatures.signature` | string | The signature of the HTTP request in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `http_signatures.algorithm` | string | The algorithm of the HTTP request signature. | `"rsa-sha256"` | -| `http_signatures.key_id` | string | The public key ID of the HTTP request signature. | `"https://example.com/actor/1#main-key"` | -| `http_signatures.verified` | boolean | Whether the HTTP request signature was verified successfully. | `false` | -| `http_signatures.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | -| `http_signatures.key_fetch_status` | int | The HTTP status code from a failed signing-key fetch, when available. | `410` | -| `http_signatures.key_fetch_error` | string | The error type from a non-HTTP signing-key fetch failure, when available. | `"TypeError"` | -| `http_signatures.digest.{algorithm}` | string | The digest of the HTTP request body in hexadecimal. The `{algorithm}` is the digest algorithm (e.g., `sha`, `sha-256`). | `"d41d8cd98f00b204e9800998ecf8427e"` | -| `ld_signatures.key_id` | string | The public key ID of the Linked Data signature. | `"https://example.com/actor/1#main-key"` | -| `ld_signatures.signature` | string | The signature of the Linked Data in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `ld_signatures.type` | string | The algorithm of the Linked Data signature. | `"RsaSignature2017"` | -| `object_integrity_proofs.cryptosuite` | string | The cryptographic suite of the object integrity proof. | `"eddsa-jcs-2022"` | -| `object_integrity_proofs.key_id` | string | The public key ID of the object integrity proof. | `"https://example.com/actor/1#main-key"` | -| `object_integrity_proofs.signature` | string | The integrity proof of the object in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `url.full` | string | The full URL being fetched by the document loader. | `"https://example.com/actor/1"` | -| `webfinger.resource` | string | The queried resource URI. | `"acct:fedify@hollo.social"` | -| `webfinger.resource.scheme` | string | The scheme of the queried resource URI. | `"acct"` | +| Attribute | Type | Description | Example | +| ----------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | +| `activitypub.activity.id` | string | The URI of the activity object. | `"https://example.com/activity/1"` | +| `activitypub.activity.type` | string[] | The qualified URI(s) of the activity type(s). | `["https://www.w3.org/ns/activitystreams#Create"]` | +| `activitypub.activity.to` | string[] | The URI(s) of the recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | +| `activitypub.activity.cc` | string[] | The URI(s) of the carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | +| `activitypub.activity.bto` | string[] | The URI(s) of the blind recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | +| `activitypub.activity.bcc` | string[] | The URI(s) of the blind carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | +| `activitypub.activity.retries` | int | The ordinal number of activity resending attempt (if and only if it's retried). | `3` | +| `activitypub.delivery.attempt` | int | The zero-based delivery attempt number for a queued outgoing activity. | `0` | +| `activitypub.delivery.permanent_failure` | boolean | Whether an outgoing delivery failure will be abandoned instead of retried. | `true` | +| `activitypub.delivery.success` | boolean | Whether an outgoing delivery attempt received a successful response. | `true` | +| `activitypub.actor.id` | string | The URI of the actor object. | `"https://example.com/actor/1"` | +| `activitypub.actor.key.cached` | boolean | Whether the actor's public keys are cached. | `true` | +| `activitypub.actor.type` | string[] | The qualified URI(s) of the actor type(s). | `["https://www.w3.org/ns/activitystreams#Person"]` | +| `activitypub.key.id` | string | The URI of the cryptographic key being verified. | `"https://example.com/actor/1#main-key"` | +| `activitypub.key_ownership.method` | string | The method used to verify key ownership (`owner_id` or `actor_fetch`). | `"actor_fetch"` | +| `activitypub.key_ownership.verified` | boolean | Whether the key ownership was successfully verified. | `true` | +| `activitypub.collection.id` | string | The URI of the collection object. | `"https://example.com/collection/1"` | +| `activitypub.collection.type` | string[] | The qualified URI(s) of the collection type(s). | `["https://www.w3.org/ns/activitystreams#OrderedCollection"]` | +| `activitypub.collection.total_items` | int | The total number of items in the collection. | `42` | +| `activitypub.object.id` | string | The URI of the object or the object enclosed by the activity. | `"https://example.com/object/1"` | +| `activitypub.object.type` | string[] | The qualified URI(s) of the object type(s). | `["https://www.w3.org/ns/activitystreams#Note"]` | +| `activitypub.object.in_reply_to` | string[] | The URI(s) of the original object to which the object reply. | `["https://example.com/object/1"]` | +| `activitypub.inboxes` | int | The number of inboxes the activity is sent to. | `12` | +| `activitypub.remote.host` | string | The hostname of the remote ActivityPub server. | `"example.com"` | +| `activitypub.shared_inbox` | boolean | Whether the activity is sent to the shared inbox. | `true` | +| `activitypub.verification.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | +| `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | +| `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | +| `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | +| `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | +| `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | +| `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | +| `fedify.collection.cursor` | string | The cursor of the collection. | `"eyJpZCI6IjEiLCJ0eXBlIjoiT3JkZXJlZENvbGxlY3Rpb24ifQ=="` | +| `fedify.collection.items` | number | The number of items in the collection page. It can be less than the total items. | `10` | +| `http.redirect.url` | string | The redirect URL when a document fetch results in a redirect. | `"https://example.com/new-location"` | +| `http.response.status_code` | int | The HTTP response status code. | `200` | +| `http_signatures.signature` | string | The signature of the HTTP request in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `http_signatures.algorithm` | string | The algorithm of the HTTP request signature. | `"rsa-sha256"` | +| `http_signatures.key_id` | string | The public key ID of the HTTP request signature. | `"https://example.com/actor/1#main-key"` | +| `http_signatures.verified` | boolean | Whether the HTTP request signature was verified successfully. | `false` | +| `http_signatures.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | +| `http_signatures.key_fetch_status` | int | The HTTP status code from a failed signing-key fetch, when available. | `410` | +| `http_signatures.key_fetch_error` | string | The error type from a non-HTTP signing-key fetch failure, when available. | `"TypeError"` | +| `http_signatures.digest.{algorithm}` | string | The digest of the HTTP request body in hexadecimal. The `{algorithm}` is the digest algorithm (e.g., `sha`, `sha-256`). | `"d41d8cd98f00b204e9800998ecf8427e"` | +| `ld_signatures.key_id` | string | The public key ID of the Linked Data signature. | `"https://example.com/actor/1#main-key"` | +| `ld_signatures.signature` | string | The signature of the Linked Data in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `ld_signatures.type` | string | The algorithm of the Linked Data signature. | `"RsaSignature2017"` | +| `object_integrity_proofs.cryptosuite` | string | The cryptographic suite of the object integrity proof. | `"eddsa-jcs-2022"` | +| `object_integrity_proofs.key_id` | string | The public key ID of the object integrity proof. | `"https://example.com/actor/1#main-key"` | +| `object_integrity_proofs.signature` | string | The integrity proof of the object in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `url.full` | string | The full URL being fetched by the document loader. | `"https://example.com/actor/1"` | +| `webfinger.resource` | string | The queried resource URI. | `"acct:fedify@hollo.social"` | +| `webfinger.resource.scheme` | string | The scheme of the queried resource URI. | `"acct"` | [attributes]: https://opentelemetry.io/docs/specs/otel/common/#attribute [OpenTelemetry Semantic Conventions]: https://opentelemetry.io/docs/specs/semconv/ diff --git a/docs/why.md b/docs/why.md index 62213af01..4e12d55f6 100644 --- a/docs/why.md +++ b/docs/why.md @@ -201,8 +201,8 @@ TypeScript-native design : Comprehensive type definitions with intelligent auto-completion Observability -: [Built-in OpenTelemetry support](./manual/opentelemetry.md) for tracing and - monitoring +: [Built-in OpenTelemetry support](./manual/opentelemetry.md) for tracing, + metrics, and monitoring [CLI toolchain](./cli.md) : Tools for testing and debugging federation, including: From 760e1494558ff73c74170b4b2f2d3fb2d0f6df70 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 5 May 2026 20:22:42 +0900 Subject: [PATCH 03/16] Document testing meter provider option Add release notes and API documentation for the mock federation meter provider option exposed by @fedify/testing. Refs: https://github.com/fedify-dev/fedify/issues/619 Assisted-by: gpt-5.5 --- CHANGES.md | 5 +++++ packages/testing/src/mock.ts | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 252bdd799..97a77b94e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -44,6 +44,11 @@ To be released. asserting OpenTelemetry metric measurements in runtime-agnostic tests. [[#619]] +### @fedify/testing + + - Added a `meterProvider` option to `createFederation()` so mock contexts can + expose a test OpenTelemetry meter provider. [[#619]] + ### @fedify/amqp - Added `AmqpMessageQueue.getDepth()` for reporting queued, ready, and diff --git a/packages/testing/src/mock.ts b/packages/testing/src/mock.ts index 12b96741c..271387c25 100644 --- a/packages/testing/src/mock.ts +++ b/packages/testing/src/mock.ts @@ -810,6 +810,10 @@ export function createFederation( options: { contextData?: TContextData; origin?: string; + /** + * The OpenTelemetry meter provider to expose from mock contexts. + * @since 2.3.0 + */ meterProvider?: any; tracerProvider?: any; } = {}, From 33f5f91f843004adf1c22e99cecae78ad6c3c28a Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 5 May 2026 21:05:09 +0900 Subject: [PATCH 04/16] Add PR links to the changelog --- CHANGES.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 97a77b94e..9ac1b9b4d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -26,28 +26,32 @@ To be released. delivery failures, inbox listener processing duration, and HTTP Signature verification failures. Applications can pass the new `meterProvider` option to `createFederation()`, and `Context.meterProvider` exposes the - provider available to request, inbox, and outbox code. [[#619]] + provider available to request, inbox, and outbox code. + [[#316], [#619], [#755]] - Added the `activitypub.delivery.failed` span event to queued outbox delivery spans so retry and permanent-failure decisions include the - remote host, attempt number, and HTTP status code when available. [[#619]] + remote host, attempt number, and HTTP status code when available. + [[#316], [#619], [#755]] +[#316]: https://github.com/fedify-dev/fedify/issues/316 [#619]: https://github.com/fedify-dev/fedify/issues/619 [#735]: https://github.com/fedify-dev/fedify/issues/735 [#748]: https://github.com/fedify-dev/fedify/pull/748 [#752]: https://github.com/fedify-dev/fedify/issues/752 [#753]: https://github.com/fedify-dev/fedify/pull/753 +[#755]: https://github.com/fedify-dev/fedify/pull/755 ### @fedify/fixture - Added `createTestMeterProvider()` and `TestMetricRecorder` helpers for asserting OpenTelemetry metric measurements in runtime-agnostic tests. - [[#619]] + [[#316], [#619], [#755]] ### @fedify/testing - Added a `meterProvider` option to `createFederation()` so mock contexts can - expose a test OpenTelemetry meter provider. [[#619]] + expose a test OpenTelemetry meter provider. [[#316], [#619], [#755]] ### @fedify/amqp From 031172db0a742838286ce141fca3e31101810043 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 5 May 2026 22:03:03 +0900 Subject: [PATCH 05/16] Harden federation metrics review fixes Make the context meter provider source-compatible for external Context implementations, count all 401 signature rejection paths, and avoid adding full activity payloads to successful delivery span events. Guard queued failure span attributes so malformed inbox payloads do not break retry logic. https://github.com/fedify-dev/fedify/pull/755#discussion_r3188308098 https://github.com/fedify-dev/fedify/pull/755#discussion_r3188308111 https://github.com/fedify-dev/fedify/pull/755#discussion_r3188308119 https://github.com/fedify-dev/fedify/pull/755#discussion_r3188308135 https://github.com/fedify-dev/fedify/pull/755#discussion_r3188308157 Assisted-by: gpt-5.5 --- docs/manual/opentelemetry.md | 14 +++---- packages/fedify/src/federation/context.ts | 2 +- .../fedify/src/federation/handler.test.ts | 40 ++++++++++++++++++- packages/fedify/src/federation/handler.ts | 10 +++++ packages/fedify/src/federation/metrics.ts | 2 +- .../fedify/src/federation/middleware.test.ts | 24 +++++++++++ packages/fedify/src/federation/middleware.ts | 14 ++++++- packages/fedify/src/federation/send.test.ts | 9 +---- packages/fedify/src/federation/send.ts | 1 - 9 files changed, 95 insertions(+), 21 deletions(-) diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index 18ef2454f..f138f7000 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -289,13 +289,13 @@ Instrumented metrics Fedify records the following OpenTelemetry metrics: -| Metric name | Instrument | Unit | Description | -| -------------------------------------------- | ---------- | ----------- | ------------------------------------------------------------- | -| `activitypub.delivery.sent` | Counter | `{attempt}` | Counts outgoing ActivityPub delivery attempts. | -| `activitypub.delivery.permanent_failure` | Counter | `{failure}` | Counts outgoing deliveries abandoned as permanent failures. | -| `activitypub.delivery.duration` | Histogram | `ms` | Measures outgoing ActivityPub delivery attempt duration. | -| `activitypub.inbox.processing_duration` | Histogram | `ms` | Measures inbox listener processing duration. | -| `activitypub.signature.verification_failure` | Counter | `{failure}` | Counts failed HTTP Signature verification for inbox requests. | +| Metric name | Instrument | Unit | Description | +| -------------------------------------------- | ---------- | ----------- | ----------------------------------------------------------- | +| `activitypub.delivery.sent` | Counter | `{attempt}` | Counts outgoing ActivityPub delivery attempts. | +| `activitypub.delivery.permanent_failure` | Counter | `{failure}` | Counts outgoing deliveries abandoned as permanent failures. | +| `activitypub.delivery.duration` | Histogram | `ms` | Measures outgoing ActivityPub delivery attempt duration. | +| `activitypub.inbox.processing_duration` | Histogram | `ms` | Measures inbox listener processing duration. | +| `activitypub.signature.verification_failure` | Counter | `{failure}` | Counts failed signature verification for inbox requests. | ### Metric attributes diff --git a/packages/fedify/src/federation/context.ts b/packages/fedify/src/federation/context.ts index ec6eafd30..14fa6a79e 100644 --- a/packages/fedify/src/federation/context.ts +++ b/packages/fedify/src/federation/context.ts @@ -74,7 +74,7 @@ export interface Context { * The OpenTelemetry meter provider. * @since 2.3.0 */ - readonly meterProvider: MeterProvider; + readonly meterProvider?: MeterProvider; /** * The document loader for loading remote JSON-LD documents. diff --git a/packages/fedify/src/federation/handler.test.ts b/packages/fedify/src/federation/handler.test.ts index ec94351f4..95dc2b144 100644 --- a/packages/fedify/src/federation/handler.test.ts +++ b/packages/fedify/src/federation/handler.test.ts @@ -3053,6 +3053,7 @@ test("handleInbox() nonce consumption on valid signed request", async () => { }); test("handleInbox() nonce replay prevention", async () => { + const [meterProvider, recorder] = createTestMeterProvider(); const activity = new Create({ id: new URL("https://example.com/activities/nonce-3"), actor: new URL("https://example.com/person2"), @@ -3075,7 +3076,10 @@ test("handleInbox() nonce replay prevention", async () => { rsaPublicKey3.id!, { spec: "rfc9421", rfc9421: { nonce } }, ); - const federation = createFederation({ kv: new MemoryKvStore() }); + const federation = createFederation({ + kv: new MemoryKvStore(), + meterProvider, + }); const context = createRequestContext({ federation, request: signedRequest, @@ -3107,6 +3111,7 @@ test("handleInbox() nonce replay prevention", async () => { onNotFound: () => new Response("Not found", { status: 404 }), signatureTimeWindow: { minutes: 5 }, skipSignatureVerification: false, + meterProvider, inboxChallengePolicy: { enabled: true, requestNonce: true, @@ -3132,6 +3137,19 @@ test("handleInbox() nonce replay prevention", async () => { "no-store", "Challenge response must have Cache-Control: no-store", ); + const failures = recorder.getMeasurements( + "activitypub.signature.verification_failure", + ); + assertEquals(failures.length, 1); + assertEquals(failures[0].value, 1); + assertEquals( + failures[0].attributes["activitypub.remote.host"], + "example.com", + ); + assertEquals( + failures[0].attributes["activitypub.verification.failure_reason"], + "invalidNonce", + ); }); test( @@ -3273,6 +3291,7 @@ test( test( "handleInbox() actor/key mismatch does not consume nonce", async () => { + const [meterProvider, recorder] = createTestMeterProvider(); // A request that has a valid RFC 9421 signature with a nonce, but the // signing key does not belong to the claimed actor. The nonce must NOT be // consumed so the legitimate sender can still use it. @@ -3304,7 +3323,10 @@ test( rsaPublicKey3.id!, { spec: "rfc9421", rfc9421: { nonce } }, ); - const federation = createFederation({ kv: new MemoryKvStore() }); + const federation = createFederation({ + kv: new MemoryKvStore(), + meterProvider, + }); const context = createRequestContext({ federation, request: maliciousRequest, @@ -3336,6 +3358,7 @@ test( onNotFound: () => new Response("Not found", { status: 404 }), signatureTimeWindow: { minutes: 5 }, skipSignatureVerification: false, + meterProvider, inboxChallengePolicy: { enabled: true, requestNonce: true, @@ -3357,6 +3380,19 @@ test( true, "Nonce must not be consumed when actor/key ownership check fails", ); + const failures = recorder.getMeasurements( + "activitypub.signature.verification_failure", + ); + assertEquals(failures.length, 1); + assertEquals(failures[0].value, 1); + assertEquals( + failures[0].attributes["activitypub.remote.host"], + "example.com", + ); + assertEquals( + failures[0].attributes["activitypub.verification.failure_reason"], + "actorKeyMismatch", + ); }, ); diff --git a/packages/fedify/src/federation/handler.ts b/packages/fedify/src/federation/handler.ts index f91846f35..71e85f686 100644 --- a/packages/fedify/src/federation/handler.ts +++ b/packages/fedify/src/federation/handler.ts @@ -1134,6 +1134,11 @@ async function handleInboxInternal( if ( httpSigKey != null && !await doesActorOwnKey(activity, httpSigKey, ctx) ) { + getFederationMetrics(parameters.meterProvider) + .recordSignatureVerificationFailure( + "actorKeyMismatch", + httpSigKey.id?.hostname, + ); logger.error( "The signer ({keyId}) and the actor ({actorId}) do not match.", { @@ -1162,6 +1167,11 @@ async function handleInboxInternal( pendingNonceLabel, ); if (!nonceValid) { + getFederationMetrics(parameters.meterProvider) + .recordSignatureVerificationFailure( + "invalidNonce", + httpSigKey?.id?.hostname, + ); logger.error( "Signature nonce verification failed (missing, expired, or replayed).", { recipient }, diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index cdf69ea84..2131ca7f6 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -30,7 +30,7 @@ class FederationMetrics { this.signatureVerificationFailure = meter.createCounter( "activitypub.signature.verification_failure", { - description: "ActivityPub HTTP Signature verification failures.", + description: "ActivityPub signature verification failures.", unit: "{failure}", }, ); diff --git a/packages/fedify/src/federation/middleware.test.ts b/packages/fedify/src/federation/middleware.test.ts index a1776349d..b39e7cea3 100644 --- a/packages/fedify/src/federation/middleware.test.ts +++ b/packages/fedify/src/federation/middleware.test.ts @@ -3657,6 +3657,30 @@ test("FederationImpl.processQueuedTask() permanent failure", async (t) => { }, ); + await t.step("malformed inbox does not break failure handling", async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + const { federation, queuedMessages } = setup({ tracerProvider }); + + await federation.processQueuedTask( + undefined, + createOutboxMessage( + "not a url", + "https://example.com/activity/9", + ["https://gone.example/users/bob"], + ), + ); + + assertEquals(queuedMessages.length, 1); + assertEquals((queuedMessages[0] as OutboxMessage).attempt, 1); + const events = exporter.getEvents( + "activitypub.outbox", + "activitypub.delivery.failed", + ); + assertEquals(events.length, 1); + assertEquals(events[0].attributes?.["activitypub.remote.host"], undefined); + assertEquals(events[0].attributes?.["activitypub.delivery.attempt"], 0); + }); + fetchMock.hardReset(); }); diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 67627ab03..7bb22d257 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -687,8 +687,20 @@ export class FederationImpl }); } catch (error) { span.setStatus({ code: SpanStatusCode.ERROR, message: String(error) }); + const remoteHost = (() => { + if (error instanceof SendActivityError) { + return getRemoteHost(error.inbox); + } + try { + return getRemoteHost(new URL(message.inbox)); + } catch (_) { + return undefined; + } + })(); span.addEvent("activitypub.delivery.failed", { - "activitypub.remote.host": getRemoteHost(new URL(message.inbox)), + ...(remoteHost == null + ? {} + : { "activitypub.remote.host": remoteHost }), "activitypub.delivery.attempt": message.attempt, "activitypub.delivery.permanent_failure": error instanceof SendActivityError && diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index ee508f373..79f81bbe8 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -498,14 +498,7 @@ test("sendActivity() records OpenTelemetry span events", async (t) => { event.attributes["activitypub.activity.id"], "https://example.com/activity", ); - assert(typeof event.attributes["activitypub.activity.json"] === "string"); - - // Verify the JSON contains the activity - const recordedActivity = JSON.parse( - event.attributes["activitypub.activity.json"] as string, - ); - assertEquals(recordedActivity.id, "https://example.com/activity"); - assertEquals(recordedActivity.type, "Create"); + assertEquals(event.attributes["activitypub.activity.json"], undefined); exporter.clear(); fetchMock.hardReset(); diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index 006212a8f..ea847be6a 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -341,7 +341,6 @@ async function sendActivityInternal( // Record the sent activity with delivery details span.addEvent("activitypub.activity.sent", { - "activitypub.activity.json": JSON.stringify(activity), "activitypub.inbox.url": inbox.href, "activitypub.activity.id": activityId ?? "", }); From 8a645d28391a9ebb79b3ea8dde769a2a07098f10 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 5 May 2026 22:43:55 +0900 Subject: [PATCH 06/16] Reuse duration helper in federation metrics Use the shared duration helper for queued inbox processing metrics and cover remote host attributes on failed delivery metric assertions. https://github.com/fedify-dev/fedify/pull/755#discussion_r3188660239 https://github.com/fedify-dev/fedify/pull/755#discussion_r3188660249 https://github.com/fedify-dev/fedify/pull/755#discussion_r3188693420 Assisted-by: gpt-5.5 --- packages/fedify/src/federation/middleware.ts | 8 ++++++-- packages/fedify/src/federation/send.test.ts | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index 7bb22d257..e690463e0 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -103,7 +103,11 @@ import { import { routeActivity } from "./inbox.ts"; import { KvKeyCache } from "./keycache.ts"; import type { KvKey, KvStore } from "./kv.ts"; -import { getFederationMetrics, getRemoteHost } from "./metrics.ts"; +import { + getDurationMs, + getFederationMetrics, + getRemoteHost, +} from "./metrics.ts"; import type { MessageQueue } from "./mq.ts"; import { acceptsJsonLd } from "./negotiation.ts"; import type { @@ -913,7 +917,7 @@ export class FederationImpl getFederationMetrics(this.meterProvider) .recordInboxProcessingDuration( activityType, - Math.max(0, performance.now() - started), + getDurationMs(started), ); } } catch (error) { diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index 79f81bbe8..4fb19c077 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -593,12 +593,20 @@ test("sendActivity() records OpenTelemetry delivery metrics", async (t) => { sent[0].attributes["activitypub.activity.type"], "https://www.w3.org/ns/activitystreams#Follow", ); + assertEquals( + sent[0].attributes["activitypub.remote.host"], + "metrics.example", + ); const durations = recorder.getMeasurements( "activitypub.delivery.duration", ); assertEquals(durations.length, 1); assertGreaterOrEqual(durations[0].value, 0); + assertEquals( + durations[0].attributes["activitypub.remote.host"], + "metrics.example", + ); recorder.clear(); fetchMock.hardReset(); From 166150d2c7000e2435e5cc51bc6cf80b299fcd46 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 5 May 2026 22:54:51 +0900 Subject: [PATCH 07/16] Clarify telemetry documentation Update the ActivityPub send event description now that sent activity JSON is no longer recorded, and list the new verification failure reasons in the semantic attribute table. https://github.com/fedify-dev/fedify/pull/755#discussion_r3188660197 https://github.com/fedify-dev/fedify/pull/755#discussion_r3188660220 Assisted-by: gpt-5.5 --- docs/manual/opentelemetry.md | 112 +++++++++++++++++------------------ 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index f138f7000..080e18def 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -233,7 +233,7 @@ The following span events are recorded: | Event name | Recorded on span | Description | | ------------------------------- | --------------------------- | -------------------------------------------------------------------------------- | | `activitypub.activity.received` | `activitypub.inbox` | Records full activity JSON and verification status when an activity is received. | -| `activitypub.activity.sent` | `activitypub.send_activity` | Records full activity JSON and delivery details when an activity is sent. | +| `activitypub.activity.sent` | `activitypub.send_activity` | Records delivery details when an activity is sent. | | `activitypub.delivery.failed` | `activitypub.outbox` | Records queued outbox delivery failure details before retry or abandonment. | | `activitypub.object.fetched` | `activitypub.lookup_object` | Records full object JSON when successfully fetched. | @@ -330,61 +330,61 @@ for ActivityPub as of November 2024. However, Fedify provides a set of semantic [attributes] for ActivityPub. The following table shows the semantic attributes for ActivityPub: -| Attribute | Type | Description | Example | -| ----------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | -| `activitypub.activity.id` | string | The URI of the activity object. | `"https://example.com/activity/1"` | -| `activitypub.activity.type` | string[] | The qualified URI(s) of the activity type(s). | `["https://www.w3.org/ns/activitystreams#Create"]` | -| `activitypub.activity.to` | string[] | The URI(s) of the recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | -| `activitypub.activity.cc` | string[] | The URI(s) of the carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | -| `activitypub.activity.bto` | string[] | The URI(s) of the blind recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | -| `activitypub.activity.bcc` | string[] | The URI(s) of the blind carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | -| `activitypub.activity.retries` | int | The ordinal number of activity resending attempt (if and only if it's retried). | `3` | -| `activitypub.delivery.attempt` | int | The zero-based delivery attempt number for a queued outgoing activity. | `0` | -| `activitypub.delivery.permanent_failure` | boolean | Whether an outgoing delivery failure will be abandoned instead of retried. | `true` | -| `activitypub.delivery.success` | boolean | Whether an outgoing delivery attempt received a successful response. | `true` | -| `activitypub.actor.id` | string | The URI of the actor object. | `"https://example.com/actor/1"` | -| `activitypub.actor.key.cached` | boolean | Whether the actor's public keys are cached. | `true` | -| `activitypub.actor.type` | string[] | The qualified URI(s) of the actor type(s). | `["https://www.w3.org/ns/activitystreams#Person"]` | -| `activitypub.key.id` | string | The URI of the cryptographic key being verified. | `"https://example.com/actor/1#main-key"` | -| `activitypub.key_ownership.method` | string | The method used to verify key ownership (`owner_id` or `actor_fetch`). | `"actor_fetch"` | -| `activitypub.key_ownership.verified` | boolean | Whether the key ownership was successfully verified. | `true` | -| `activitypub.collection.id` | string | The URI of the collection object. | `"https://example.com/collection/1"` | -| `activitypub.collection.type` | string[] | The qualified URI(s) of the collection type(s). | `["https://www.w3.org/ns/activitystreams#OrderedCollection"]` | -| `activitypub.collection.total_items` | int | The total number of items in the collection. | `42` | -| `activitypub.object.id` | string | The URI of the object or the object enclosed by the activity. | `"https://example.com/object/1"` | -| `activitypub.object.type` | string[] | The qualified URI(s) of the object type(s). | `["https://www.w3.org/ns/activitystreams#Note"]` | -| `activitypub.object.in_reply_to` | string[] | The URI(s) of the original object to which the object reply. | `["https://example.com/object/1"]` | -| `activitypub.inboxes` | int | The number of inboxes the activity is sent to. | `12` | -| `activitypub.remote.host` | string | The hostname of the remote ActivityPub server. | `"example.com"` | -| `activitypub.shared_inbox` | boolean | Whether the activity is sent to the shared inbox. | `true` | -| `activitypub.verification.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | -| `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | -| `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | -| `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | -| `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | -| `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | -| `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | -| `fedify.collection.cursor` | string | The cursor of the collection. | `"eyJpZCI6IjEiLCJ0eXBlIjoiT3JkZXJlZENvbGxlY3Rpb24ifQ=="` | -| `fedify.collection.items` | number | The number of items in the collection page. It can be less than the total items. | `10` | -| `http.redirect.url` | string | The redirect URL when a document fetch results in a redirect. | `"https://example.com/new-location"` | -| `http.response.status_code` | int | The HTTP response status code. | `200` | -| `http_signatures.signature` | string | The signature of the HTTP request in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `http_signatures.algorithm` | string | The algorithm of the HTTP request signature. | `"rsa-sha256"` | -| `http_signatures.key_id` | string | The public key ID of the HTTP request signature. | `"https://example.com/actor/1#main-key"` | -| `http_signatures.verified` | boolean | Whether the HTTP request signature was verified successfully. | `false` | -| `http_signatures.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | -| `http_signatures.key_fetch_status` | int | The HTTP status code from a failed signing-key fetch, when available. | `410` | -| `http_signatures.key_fetch_error` | string | The error type from a non-HTTP signing-key fetch failure, when available. | `"TypeError"` | -| `http_signatures.digest.{algorithm}` | string | The digest of the HTTP request body in hexadecimal. The `{algorithm}` is the digest algorithm (e.g., `sha`, `sha-256`). | `"d41d8cd98f00b204e9800998ecf8427e"` | -| `ld_signatures.key_id` | string | The public key ID of the Linked Data signature. | `"https://example.com/actor/1#main-key"` | -| `ld_signatures.signature` | string | The signature of the Linked Data in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `ld_signatures.type` | string | The algorithm of the Linked Data signature. | `"RsaSignature2017"` | -| `object_integrity_proofs.cryptosuite` | string | The cryptographic suite of the object integrity proof. | `"eddsa-jcs-2022"` | -| `object_integrity_proofs.key_id` | string | The public key ID of the object integrity proof. | `"https://example.com/actor/1#main-key"` | -| `object_integrity_proofs.signature` | string | The integrity proof of the object in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `url.full` | string | The full URL being fetched by the document loader. | `"https://example.com/actor/1"` | -| `webfinger.resource` | string | The queried resource URI. | `"acct:fedify@hollo.social"` | -| `webfinger.resource.scheme` | string | The scheme of the queried resource URI. | `"acct"` | +| Attribute | Type | Description | Example | +| ----------------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | +| `activitypub.activity.id` | string | The URI of the activity object. | `"https://example.com/activity/1"` | +| `activitypub.activity.type` | string[] | The qualified URI(s) of the activity type(s). | `["https://www.w3.org/ns/activitystreams#Create"]` | +| `activitypub.activity.to` | string[] | The URI(s) of the recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | +| `activitypub.activity.cc` | string[] | The URI(s) of the carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | +| `activitypub.activity.bto` | string[] | The URI(s) of the blind recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | +| `activitypub.activity.bcc` | string[] | The URI(s) of the blind carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | +| `activitypub.activity.retries` | int | The ordinal number of activity resending attempt (if and only if it's retried). | `3` | +| `activitypub.delivery.attempt` | int | The zero-based delivery attempt number for a queued outgoing activity. | `0` | +| `activitypub.delivery.permanent_failure` | boolean | Whether an outgoing delivery failure will be abandoned instead of retried. | `true` | +| `activitypub.delivery.success` | boolean | Whether an outgoing delivery attempt received a successful response. | `true` | +| `activitypub.actor.id` | string | The URI of the actor object. | `"https://example.com/actor/1"` | +| `activitypub.actor.key.cached` | boolean | Whether the actor's public keys are cached. | `true` | +| `activitypub.actor.type` | string[] | The qualified URI(s) of the actor type(s). | `["https://www.w3.org/ns/activitystreams#Person"]` | +| `activitypub.key.id` | string | The URI of the cryptographic key being verified. | `"https://example.com/actor/1#main-key"` | +| `activitypub.key_ownership.method` | string | The method used to verify key ownership (`owner_id` or `actor_fetch`). | `"actor_fetch"` | +| `activitypub.key_ownership.verified` | boolean | Whether the key ownership was successfully verified. | `true` | +| `activitypub.collection.id` | string | The URI of the collection object. | `"https://example.com/collection/1"` | +| `activitypub.collection.type` | string[] | The qualified URI(s) of the collection type(s). | `["https://www.w3.org/ns/activitystreams#OrderedCollection"]` | +| `activitypub.collection.total_items` | int | The total number of items in the collection. | `42` | +| `activitypub.object.id` | string | The URI of the object or the object enclosed by the activity. | `"https://example.com/object/1"` | +| `activitypub.object.type` | string[] | The qualified URI(s) of the object type(s). | `["https://www.w3.org/ns/activitystreams#Note"]` | +| `activitypub.object.in_reply_to` | string[] | The URI(s) of the original object to which the object reply. | `["https://example.com/object/1"]` | +| `activitypub.inboxes` | int | The number of inboxes the activity is sent to. | `12` | +| `activitypub.remote.host` | string | The hostname of the remote ActivityPub server. | `"example.com"` | +| `activitypub.shared_inbox` | boolean | Whether the activity is sent to the shared inbox. | `true` | +| `activitypub.verification.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, `keyFetchError`, `actorKeyMismatch`, or `invalidNonce`). | `"keyFetchError"` | +| `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | +| `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | +| `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | +| `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | +| `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | +| `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | +| `fedify.collection.cursor` | string | The cursor of the collection. | `"eyJpZCI6IjEiLCJ0eXBlIjoiT3JkZXJlZENvbGxlY3Rpb24ifQ=="` | +| `fedify.collection.items` | number | The number of items in the collection page. It can be less than the total items. | `10` | +| `http.redirect.url` | string | The redirect URL when a document fetch results in a redirect. | `"https://example.com/new-location"` | +| `http.response.status_code` | int | The HTTP response status code. | `200` | +| `http_signatures.signature` | string | The signature of the HTTP request in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `http_signatures.algorithm` | string | The algorithm of the HTTP request signature. | `"rsa-sha256"` | +| `http_signatures.key_id` | string | The public key ID of the HTTP request signature. | `"https://example.com/actor/1#main-key"` | +| `http_signatures.verified` | boolean | Whether the HTTP request signature was verified successfully. | `false` | +| `http_signatures.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | +| `http_signatures.key_fetch_status` | int | The HTTP status code from a failed signing-key fetch, when available. | `410` | +| `http_signatures.key_fetch_error` | string | The error type from a non-HTTP signing-key fetch failure, when available. | `"TypeError"` | +| `http_signatures.digest.{algorithm}` | string | The digest of the HTTP request body in hexadecimal. The `{algorithm}` is the digest algorithm (e.g., `sha`, `sha-256`). | `"d41d8cd98f00b204e9800998ecf8427e"` | +| `ld_signatures.key_id` | string | The public key ID of the Linked Data signature. | `"https://example.com/actor/1#main-key"` | +| `ld_signatures.signature` | string | The signature of the Linked Data in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `ld_signatures.type` | string | The algorithm of the Linked Data signature. | `"RsaSignature2017"` | +| `object_integrity_proofs.cryptosuite` | string | The cryptographic suite of the object integrity proof. | `"eddsa-jcs-2022"` | +| `object_integrity_proofs.key_id` | string | The public key ID of the object integrity proof. | `"https://example.com/actor/1#main-key"` | +| `object_integrity_proofs.signature` | string | The integrity proof of the object in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `url.full` | string | The full URL being fetched by the document loader. | `"https://example.com/actor/1"` | +| `webfinger.resource` | string | The queried resource URI. | `"acct:fedify@hollo.social"` | +| `webfinger.resource.scheme` | string | The scheme of the queried resource URI. | `"acct"` | [attributes]: https://opentelemetry.io/docs/specs/otel/common/#attribute [OpenTelemetry Semantic Conventions]: https://opentelemetry.io/docs/specs/semconv/ From bb03d95ce21dced30c552a7bd299b06a7bb30cec Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Tue, 5 May 2026 23:48:36 +0900 Subject: [PATCH 08/16] Align delivery telemetry attributes Record delivery success on delivery duration metrics so successful and failed attempts can be separated in histograms, and remove the stale sent activity JSON attribute from the OpenTelemetry docs. https://github.com/fedify-dev/fedify/pull/755#discussion_r3188994636 https://github.com/fedify-dev/fedify/pull/755#discussion_r3189078186 Assisted-by: gpt-5.5 --- docs/manual/opentelemetry.md | 3 +-- packages/fedify/src/federation/metrics.ts | 1 + packages/fedify/src/federation/send.test.ts | 8 ++++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index 080e18def..bd0c80c7f 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -263,7 +263,6 @@ Each span event includes attributes with detailed information: **`activitypub.activity.sent` event attributes:** - - `activitypub.activity.json`: The complete activity JSON being sent - `activitypub.inbox.url`: The inbox URL where the activity was delivered - `activitypub.activity.id`: The activity ID @@ -307,7 +306,7 @@ Fedify records the following OpenTelemetry metrics: : `activitypub.remote.host` and `http.response.status_code`. `activitypub.delivery.duration` -: `activitypub.remote.host`. +: `activitypub.remote.host` and `activitypub.delivery.success`. `activitypub.inbox.processing_duration` : `activitypub.activity.type`. diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index 2131ca7f6..83169b250 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -66,6 +66,7 @@ class FederationMetrics { this.deliverySent.add(1, deliveryAttributes); this.deliveryDuration.record(durationMs, { "activitypub.remote.host": getRemoteHost(inbox), + "activitypub.delivery.success": success, }); } diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index 4fb19c077..a70fea239 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -554,6 +554,10 @@ test("sendActivity() records OpenTelemetry delivery metrics", async (t) => { durations[0].attributes["activitypub.remote.host"], "metrics.example", ); + assertEquals( + durations[0].attributes["activitypub.delivery.success"], + true, + ); recorder.clear(); fetchMock.hardReset(); @@ -607,6 +611,10 @@ test("sendActivity() records OpenTelemetry delivery metrics", async (t) => { durations[0].attributes["activitypub.remote.host"], "metrics.example", ); + assertEquals( + durations[0].attributes["activitypub.delivery.success"], + false, + ); recorder.clear(); fetchMock.hardReset(); From da3b9857f8aa2259e27d1e3382603475f2a03740 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 6 May 2026 01:15:35 +0900 Subject: [PATCH 09/16] Update outbound telemetry debug example The debug exporter example now records the outbound activity ID and inbox URL from the sent activity event attributes instead of parsing the removed activity JSON event attribute. Assisted-by: gpt-5.5 --- docs/manual/opentelemetry.md | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index bd0c80c7f..76edb68f6 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -408,13 +408,22 @@ ActivityPub activities for a debug dashboard: import type { SpanExporter, ReadableSpan } from "@opentelemetry/sdk-trace-base"; import { ExportResultCode } from "@opentelemetry/core"; -interface ActivityRecord { - direction: "inbound" | "outbound"; +interface InboundActivityRecord { + direction: "inbound"; activity: unknown; timestamp: Date; verified?: boolean; } +interface OutboundActivityRecord { + direction: "outbound"; + activityId?: string; + inboxUrl?: string; + timestamp: Date; +} + +type ActivityRecord = InboundActivityRecord | OutboundActivityRecord; + export class FedifyDebugExporter implements SpanExporter { private activities: ActivityRecord[] = []; @@ -443,11 +452,16 @@ export class FedifyDebugExporter implements SpanExporter { (e) => e.name === "activitypub.activity.sent" ); if (event && event.attributes) { + const activityId = event.attributes[ + "activitypub.activity.id" + ] as string | undefined; + const inboxUrl = event.attributes[ + "activitypub.inbox.url" + ] as string | undefined; this.activities.push({ direction: "outbound", - activity: JSON.parse( - event.attributes["activitypub.activity.json"] as string - ), + activityId, + inboxUrl, timestamp: new Date(span.startTime[0] * 1000), }); } From 876e81c99f85b6ed0d78dc1828a94f84762fc3e9 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 6 May 2026 02:51:43 +0900 Subject: [PATCH 10/16] Harden delivery metrics tests Add fallback metadata for the federation meter and make the SDK metrics test always release its meter provider and fetch mock, even when assertions fail. https://github.com/fedify-dev/fedify/pull/755#discussion_r3189414945 https://github.com/fedify-dev/fedify/pull/755#discussion_r3189459667 Assisted-by: gpt-5.5 --- packages/fedify/src/federation/metrics.ts | 5 +- packages/fedify/src/federation/send.test.ts | 63 +++++++++++---------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index 83169b250..7886a28b1 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -7,6 +7,9 @@ import { } from "@opentelemetry/api"; import metadata from "../../deno.json" with { type: "json" }; +const meterName = metadata.name || "@fedify/fedify"; +const meterVersion = metadata.version || undefined; + class FederationMetrics { readonly deliverySent: Counter; readonly deliveryPermanentFailure: Counter; @@ -15,7 +18,7 @@ class FederationMetrics { readonly inboxProcessingDuration: Histogram; constructor(meterProvider: MeterProvider) { - const meter = meterProvider.getMeter(metadata.name, metadata.version); + const meter = meterProvider.getMeter(meterName, meterVersion); this.deliverySent = meter.createCounter("activitypub.delivery.sent", { description: "ActivityPub delivery attempts.", unit: "{attempt}", diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index a70fea239..7f796aca3 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -633,34 +633,39 @@ test("sendActivity() exports delivery metrics through OpenTelemetry SDK", async fetchMock.spyGlobal(); fetchMock.post("https://sdk-metrics.example/inbox", { status: 202 }); - await sendActivity({ - activity: { - "@context": "https://www.w3.org/ns/activitystreams", - type: "Create", - id: "https://example.com/activity", - actor: "https://example.com/person", - }, - activityId: "https://example.com/activity", - activityType: "https://www.w3.org/ns/activitystreams#Create", - keys: [], - inbox: new URL("https://sdk-metrics.example/inbox"), - meterProvider, - }); - - await meterProvider.forceFlush(); - const exportedMetrics = exporter.getMetrics() - .flatMap((resourceMetrics) => resourceMetrics.scopeMetrics) - .flatMap((scopeMetrics) => scopeMetrics.metrics); - const sent = exportedMetrics.find((metric) => - metric.descriptor.name === "activitypub.delivery.sent" - ); - assert(sent != null); - assertEquals(sent.dataPoints.length, 1); - assertEquals( - sent.dataPoints[0].attributes["activitypub.remote.host"], - "sdk-metrics.example", - ); + try { + await sendActivity({ + activity: { + "@context": "https://www.w3.org/ns/activitystreams", + type: "Create", + id: "https://example.com/activity", + actor: "https://example.com/person", + }, + activityId: "https://example.com/activity", + activityType: "https://www.w3.org/ns/activitystreams#Create", + keys: [], + inbox: new URL("https://sdk-metrics.example/inbox"), + meterProvider, + }); - await meterProvider.shutdown(); - fetchMock.hardReset(); + await meterProvider.forceFlush(); + const exportedMetrics = exporter.getMetrics() + .flatMap((resourceMetrics) => resourceMetrics.scopeMetrics) + .flatMap((scopeMetrics) => scopeMetrics.metrics); + const sent = exportedMetrics.find((metric) => + metric.descriptor.name === "activitypub.delivery.sent" + ); + assert(sent != null); + assertEquals(sent.dataPoints.length, 1); + assertEquals( + sent.dataPoints[0].attributes["activitypub.remote.host"], + "sdk-metrics.example", + ); + } finally { + try { + await meterProvider.shutdown(); + } finally { + fetchMock.hardReset(); + } + } }); From feb4a86b67c4b14589b63b0fce20dbe7222a8d42 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 6 May 2026 11:39:22 +0900 Subject: [PATCH 11/16] Reuse remote host helper in inbox metrics Route signature verification failure metrics through the shared remote host helper so inbox rejection paths use the same host extraction as delivery metrics. https://github.com/fedify-dev/fedify/pull/755#discussion_r3190522441 https://github.com/fedify-dev/fedify/pull/755#discussion_r3190522448 https://github.com/fedify-dev/fedify/pull/755#discussion_r3190522456 Assisted-by: gpt-5.5 --- packages/fedify/src/federation/handler.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/fedify/src/federation/handler.ts b/packages/fedify/src/federation/handler.ts index 71e85f686..5f6dc0ed9 100644 --- a/packages/fedify/src/federation/handler.ts +++ b/packages/fedify/src/federation/handler.ts @@ -64,7 +64,7 @@ import type { import { routeActivity } from "./inbox.ts"; import { KvKeyCache } from "./keycache.ts"; import type { KvKey, KvStore } from "./kv.ts"; -import { getFederationMetrics } from "./metrics.ts"; +import { getFederationMetrics, getRemoteHost } from "./metrics.ts"; import type { MessageQueue } from "./mq.ts"; import { acceptsJsonLd } from "./negotiation.ts"; @@ -995,8 +995,8 @@ async function handleInboxInternal( }); if (verification.verified === false) { const reason = verification.reason; - const remoteHost = "keyId" in reason - ? reason.keyId?.hostname + const remoteHost = "keyId" in reason && reason.keyId != null + ? getRemoteHost(reason.keyId) : undefined; getFederationMetrics(parameters.meterProvider) .recordSignatureVerificationFailure(reason.type, remoteHost); @@ -1137,7 +1137,7 @@ async function handleInboxInternal( getFederationMetrics(parameters.meterProvider) .recordSignatureVerificationFailure( "actorKeyMismatch", - httpSigKey.id?.hostname, + httpSigKey.id == null ? undefined : getRemoteHost(httpSigKey.id), ); logger.error( "The signer ({keyId}) and the actor ({actorId}) do not match.", @@ -1170,7 +1170,7 @@ async function handleInboxInternal( getFederationMetrics(parameters.meterProvider) .recordSignatureVerificationFailure( "invalidNonce", - httpSigKey?.id?.hostname, + httpSigKey?.id == null ? undefined : getRemoteHost(httpSigKey.id), ); logger.error( "Signature nonce verification failed (missing, expired, or replayed).", From f9931640074688501239d50fca86467a37ed8f9b Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 6 May 2026 12:15:50 +0900 Subject: [PATCH 12/16] Warn on malformed queued inbox URLs Queue messages with invalid inbox URLs already keep failure telemetry from crashing. Log the malformed inbox too, so operators can identify corrupted outbox messages. https://github.com/fedify-dev/fedify/pull/755#discussion_r3192694466 Assisted-by: gpt-5.5 --- .../fedify/src/federation/middleware.test.ts | 66 +++++++++++++------ packages/fedify/src/federation/middleware.ts | 4 ++ 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/packages/fedify/src/federation/middleware.test.ts b/packages/fedify/src/federation/middleware.test.ts index b39e7cea3..681a52805 100644 --- a/packages/fedify/src/federation/middleware.test.ts +++ b/packages/fedify/src/federation/middleware.test.ts @@ -3658,27 +3658,55 @@ test("FederationImpl.processQueuedTask() permanent failure", async (t) => { ); await t.step("malformed inbox does not break failure handling", async () => { - const [tracerProvider, exporter] = createTestTracerProvider(); - const { federation, queuedMessages } = setup({ tracerProvider }); + await withLogtapeLock(async () => { + const [tracerProvider, exporter] = createTestTracerProvider(); + const { federation, queuedMessages } = setup({ tracerProvider }); + const records: LogRecord[] = []; + await reset(); + try { + await configure({ + sinks: { + buffer(record: LogRecord): void { + records.push(record); + }, + }, + filters: {}, + loggers: [{ category: [], sinks: ["buffer"] }], + }); - await federation.processQueuedTask( - undefined, - createOutboxMessage( - "not a url", - "https://example.com/activity/9", - ["https://gone.example/users/bob"], - ), - ); + await federation.processQueuedTask( + undefined, + createOutboxMessage( + "not a url", + "https://example.com/activity/9", + ["https://gone.example/users/bob"], + ), + ); - assertEquals(queuedMessages.length, 1); - assertEquals((queuedMessages[0] as OutboxMessage).attempt, 1); - const events = exporter.getEvents( - "activitypub.outbox", - "activitypub.delivery.failed", - ); - assertEquals(events.length, 1); - assertEquals(events[0].attributes?.["activitypub.remote.host"], undefined); - assertEquals(events[0].attributes?.["activitypub.delivery.attempt"], 0); + assertEquals(queuedMessages.length, 1); + assertEquals((queuedMessages[0] as OutboxMessage).attempt, 1); + const events = exporter.getEvents( + "activitypub.outbox", + "activitypub.delivery.failed", + ); + assertEquals(events.length, 1); + assertEquals( + events[0].attributes?.["activitypub.remote.host"], + undefined, + ); + assertEquals(events[0].attributes?.["activitypub.delivery.attempt"], 0); + assertEquals( + records.some((record) => + record.rawMessage === + "Invalid inbox URL in queued outbox message: {inbox}" && + record.properties.inbox === "not a url" + ), + true, + ); + } finally { + await reset(); + } + }); }); fetchMock.hardReset(); diff --git a/packages/fedify/src/federation/middleware.ts b/packages/fedify/src/federation/middleware.ts index e690463e0..3ccbcba1a 100644 --- a/packages/fedify/src/federation/middleware.ts +++ b/packages/fedify/src/federation/middleware.ts @@ -698,6 +698,10 @@ export class FederationImpl try { return getRemoteHost(new URL(message.inbox)); } catch (_) { + logger.warn( + "Invalid inbox URL in queued outbox message: {inbox}", + logData, + ); return undefined; } })(); From 8e5eb6485376038134c3eb55780267e29ef64f43 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 6 May 2026 15:47:01 +0900 Subject: [PATCH 13/16] Align outbound trace activity records Outbound send activity events now carry delivery metadata instead of full activity JSON. Let FedifySpanExporter store those records from the available activity ID and inbox URL, and hide the debugger JSON panel when a record has no captured payload. https://github.com/fedify-dev/fedify/pull/755#discussion_r3192793920 Assisted-by: gpt-5.5 --- CHANGES.md | 7 ++++ docs/manual/opentelemetry.md | 5 +++ packages/debugger/src/views/trace-detail.tsx | 10 +++-- packages/fedify/src/otel/exporter.test.ts | 39 ++++++++++---------- packages/fedify/src/otel/exporter.ts | 36 +++++++++--------- 5 files changed, 57 insertions(+), 40 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9ac1b9b4d..c57a58599 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -34,6 +34,13 @@ To be released. remote host, attempt number, and HTTP status code when available. [[#316], [#619], [#755]] + - *Breaking change*: Changed the `activitypub.activity.sent` span event to + record delivery metadata (`activitypub.inbox.url` and + `activitypub.activity.id`) instead of the full `activitypub.activity.json` + payload. `FedifySpanExporter` now stores outbound records from those + attributes, and `TraceActivityRecord.activityJson` is present only when the + span event includes full activity JSON. [[#316], [#619], [#755]] + [#316]: https://github.com/fedify-dev/fedify/issues/316 [#619]: https://github.com/fedify-dev/fedify/issues/619 [#735]: https://github.com/fedify-dev/fedify/issues/735 diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index 76edb68f6..7490d1e4b 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -266,6 +266,11 @@ Each span event includes attributes with detailed information: - `activitypub.inbox.url`: The inbox URL where the activity was delivered - `activitypub.activity.id`: The activity ID +The `activitypub.activity.sent` event records delivery metadata only. It does +not include the full `activitypub.activity.json` payload; if you need the full +outbound activity for auditing, store it in your application before delivery +and correlate it with `activitypub.activity.id`. + **`activitypub.delivery.failed` event attributes:** - `activitypub.remote.host`: The remote inbox host diff --git a/packages/debugger/src/views/trace-detail.tsx b/packages/debugger/src/views/trace-detail.tsx index 22f2899bc..2277709a3 100644 --- a/packages/debugger/src/views/trace-detail.tsx +++ b/packages/debugger/src/views/trace-detail.tsx @@ -168,10 +168,12 @@ export const TraceDetailPage: FC = ( -
- Activity JSON - 
{formatJson(activity.activityJson)}
-
+ {activity.activityJson == null ? null : ( +
+ Activity JSON + 
{formatJson(activity.activityJson)}
+
+ )} )) )} diff --git a/packages/fedify/src/otel/exporter.test.ts b/packages/fedify/src/otel/exporter.test.ts index 2b583d108..58e1bf710 100644 --- a/packages/fedify/src/otel/exporter.test.ts +++ b/packages/fedify/src/otel/exporter.test.ts @@ -1,5 +1,10 @@ import { test } from "@fedify/fixture"; -import type { HrTime, SpanContext, SpanStatus } from "@opentelemetry/api"; +import type { + Attributes, + HrTime, + SpanContext, + SpanStatus, +} from "@opentelemetry/api"; import { SpanKind, SpanStatusCode, TraceFlags } from "@opentelemetry/api"; import type { ReadableSpan, TimedEvent } from "@opentelemetry/sdk-trace-base"; import { assertEquals } from "@std/assert"; @@ -91,18 +96,21 @@ function createActivityReceivedEvent(options: { } function createActivitySentEvent(options: { - activityJson: string; + activityJson?: string; inboxUrl: string; activityId?: string; }): TimedEvent { + const attributes: Attributes = { + "activitypub.inbox.url": options.inboxUrl, + "activitypub.activity.id": options.activityId ?? "", + }; + if (options.activityJson != null) { + attributes["activitypub.activity.json"] = options.activityJson; + } return { name: "activitypub.activity.sent", time: [1700000000, 500000000] as HrTime, - attributes: { - "activitypub.activity.json": options.activityJson, - "activitypub.inbox.url": options.inboxUrl, - "activitypub.activity.id": options.activityId ?? "", - }, + attributes, }; } @@ -171,14 +179,7 @@ test("FedifySpanExporter", async (t) => { const traceId = "trace789"; const spanId = "span012"; const inboxUrl = "https://example.com/users/alice/inbox"; - const activity = { - "@context": "https://www.w3.org/ns/activitystreams", - type: "Follow", - id: "https://myserver.com/activities/789", - actor: "https://myserver.com/users/bob", - object: "https://example.com/users/alice", - }; - const activityJson = JSON.stringify(activity); + const activityId = "https://myserver.com/activities/789"; const span = createMockSpan({ traceId, @@ -186,9 +187,8 @@ test("FedifySpanExporter", async (t) => { name: "activitypub.send_activity", events: [ createActivitySentEvent({ - activityJson, inboxUrl, - activityId: activity.id, + activityId, }), ], }); @@ -205,8 +205,9 @@ test("FedifySpanExporter", async (t) => { assertEquals(activities[0].traceId, traceId); assertEquals(activities[0].spanId, spanId); assertEquals(activities[0].direction, "outbound"); - assertEquals(activities[0].activityType, activity.type); - assertEquals(activities[0].activityId, activity.id); + assertEquals(activities[0].activityType, "Unknown"); + assertEquals(activities[0].activityId, activityId); + assertEquals(activities[0].activityJson, undefined); assertEquals(activities[0].inboxUrl, inboxUrl); }, ); diff --git a/packages/fedify/src/otel/exporter.ts b/packages/fedify/src/otel/exporter.ts index 685ff0940..8c87692d3 100644 --- a/packages/fedify/src/otel/exporter.ts +++ b/packages/fedify/src/otel/exporter.ts @@ -91,9 +91,10 @@ export interface TraceActivityRecord { readonly actorId?: string; /** - * The full JSON representation of the activity. + * The full JSON representation of the activity, if the span event included + * it. */ - readonly activityJson: string; + readonly activityJson?: string; /** * Whether the activity was verified (for inbound activities). @@ -406,26 +407,27 @@ export class FedifySpanExporter implements SpanExporter { if (attrs == null) return null; const activityJson = attrs["activitypub.activity.json"]; - if (typeof activityJson !== "string") return null; let activityType = "Unknown"; let activityId: string | undefined; let actorId: string | undefined; - try { - const activity = JSON.parse(activityJson); - activityType = activity.type ?? "Unknown"; - activityId = activity.id; - // Extract actor ID from activity - if (typeof activity.actor === "string") { - actorId = activity.actor; - } else if ( - activity.actor != null && typeof activity.actor.id === "string" - ) { - actorId = activity.actor.id; + if (typeof activityJson === "string") { + try { + const activity = JSON.parse(activityJson); + activityType = activity.type ?? "Unknown"; + activityId = activity.id; + // Extract actor ID from activity + if (typeof activity.actor === "string") { + actorId = activity.actor; + } else if ( + activity.actor != null && typeof activity.actor.id === "string" + ) { + actorId = activity.actor.id; + } + } catch { + // Ignore JSON parse errors } - } catch { - // Ignore JSON parse errors } const inboxUrl = attrs["activitypub.inbox.url"]; @@ -442,7 +444,7 @@ export class FedifySpanExporter implements SpanExporter { ? explicitActivityId : undefined), actorId, - activityJson, + ...(typeof activityJson === "string" ? { activityJson } : {}), timestamp: new Date( event.time[0] * 1000 + event.time[1] / 1e6, ).toISOString(), From 7b2bf8b07205fc0153114bfad368e984398c6290 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Wed, 6 May 2026 17:31:01 +0900 Subject: [PATCH 14/16] Align delivery metric attributes Delivery attempt duration metrics now carry the same activity type attribute as the delivery counter when Fedify knows the activity type. The OpenTelemetry docs also keep metric-only attributes out of the span semantic attribute table. https://github.com/fedify-dev/fedify/pull/755#discussion_r3193556989 https://github.com/fedify-dev/fedify/pull/755#discussion_r3193556996 Assisted-by: gpt-5.5 --- docs/manual/opentelemetry.md | 111 ++++++++++---------- packages/fedify/src/federation/metrics.ts | 5 +- packages/fedify/src/federation/send.test.ts | 8 ++ 3 files changed, 64 insertions(+), 60 deletions(-) diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index 7490d1e4b..14892453c 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -311,7 +311,8 @@ Fedify records the following OpenTelemetry metrics: : `activitypub.remote.host` and `http.response.status_code`. `activitypub.delivery.duration` -: `activitypub.remote.host` and `activitypub.delivery.success`. +: `activitypub.remote.host`, `activitypub.delivery.success`, and + `activitypub.activity.type` when Fedify knows the activity type. `activitypub.inbox.processing_duration` : `activitypub.activity.type`. @@ -334,61 +335,59 @@ for ActivityPub as of November 2024. However, Fedify provides a set of semantic [attributes] for ActivityPub. The following table shows the semantic attributes for ActivityPub: -| Attribute | Type | Description | Example | -| ----------------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | -| `activitypub.activity.id` | string | The URI of the activity object. | `"https://example.com/activity/1"` | -| `activitypub.activity.type` | string[] | The qualified URI(s) of the activity type(s). | `["https://www.w3.org/ns/activitystreams#Create"]` | -| `activitypub.activity.to` | string[] | The URI(s) of the recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | -| `activitypub.activity.cc` | string[] | The URI(s) of the carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | -| `activitypub.activity.bto` | string[] | The URI(s) of the blind recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | -| `activitypub.activity.bcc` | string[] | The URI(s) of the blind carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | -| `activitypub.activity.retries` | int | The ordinal number of activity resending attempt (if and only if it's retried). | `3` | -| `activitypub.delivery.attempt` | int | The zero-based delivery attempt number for a queued outgoing activity. | `0` | -| `activitypub.delivery.permanent_failure` | boolean | Whether an outgoing delivery failure will be abandoned instead of retried. | `true` | -| `activitypub.delivery.success` | boolean | Whether an outgoing delivery attempt received a successful response. | `true` | -| `activitypub.actor.id` | string | The URI of the actor object. | `"https://example.com/actor/1"` | -| `activitypub.actor.key.cached` | boolean | Whether the actor's public keys are cached. | `true` | -| `activitypub.actor.type` | string[] | The qualified URI(s) of the actor type(s). | `["https://www.w3.org/ns/activitystreams#Person"]` | -| `activitypub.key.id` | string | The URI of the cryptographic key being verified. | `"https://example.com/actor/1#main-key"` | -| `activitypub.key_ownership.method` | string | The method used to verify key ownership (`owner_id` or `actor_fetch`). | `"actor_fetch"` | -| `activitypub.key_ownership.verified` | boolean | Whether the key ownership was successfully verified. | `true` | -| `activitypub.collection.id` | string | The URI of the collection object. | `"https://example.com/collection/1"` | -| `activitypub.collection.type` | string[] | The qualified URI(s) of the collection type(s). | `["https://www.w3.org/ns/activitystreams#OrderedCollection"]` | -| `activitypub.collection.total_items` | int | The total number of items in the collection. | `42` | -| `activitypub.object.id` | string | The URI of the object or the object enclosed by the activity. | `"https://example.com/object/1"` | -| `activitypub.object.type` | string[] | The qualified URI(s) of the object type(s). | `["https://www.w3.org/ns/activitystreams#Note"]` | -| `activitypub.object.in_reply_to` | string[] | The URI(s) of the original object to which the object reply. | `["https://example.com/object/1"]` | -| `activitypub.inboxes` | int | The number of inboxes the activity is sent to. | `12` | -| `activitypub.remote.host` | string | The hostname of the remote ActivityPub server. | `"example.com"` | -| `activitypub.shared_inbox` | boolean | Whether the activity is sent to the shared inbox. | `true` | -| `activitypub.verification.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, `keyFetchError`, `actorKeyMismatch`, or `invalidNonce`). | `"keyFetchError"` | -| `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | -| `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | -| `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | -| `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | -| `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | -| `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | -| `fedify.collection.cursor` | string | The cursor of the collection. | `"eyJpZCI6IjEiLCJ0eXBlIjoiT3JkZXJlZENvbGxlY3Rpb24ifQ=="` | -| `fedify.collection.items` | number | The number of items in the collection page. It can be less than the total items. | `10` | -| `http.redirect.url` | string | The redirect URL when a document fetch results in a redirect. | `"https://example.com/new-location"` | -| `http.response.status_code` | int | The HTTP response status code. | `200` | -| `http_signatures.signature` | string | The signature of the HTTP request in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `http_signatures.algorithm` | string | The algorithm of the HTTP request signature. | `"rsa-sha256"` | -| `http_signatures.key_id` | string | The public key ID of the HTTP request signature. | `"https://example.com/actor/1#main-key"` | -| `http_signatures.verified` | boolean | Whether the HTTP request signature was verified successfully. | `false` | -| `http_signatures.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | -| `http_signatures.key_fetch_status` | int | The HTTP status code from a failed signing-key fetch, when available. | `410` | -| `http_signatures.key_fetch_error` | string | The error type from a non-HTTP signing-key fetch failure, when available. | `"TypeError"` | -| `http_signatures.digest.{algorithm}` | string | The digest of the HTTP request body in hexadecimal. The `{algorithm}` is the digest algorithm (e.g., `sha`, `sha-256`). | `"d41d8cd98f00b204e9800998ecf8427e"` | -| `ld_signatures.key_id` | string | The public key ID of the Linked Data signature. | `"https://example.com/actor/1#main-key"` | -| `ld_signatures.signature` | string | The signature of the Linked Data in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `ld_signatures.type` | string | The algorithm of the Linked Data signature. | `"RsaSignature2017"` | -| `object_integrity_proofs.cryptosuite` | string | The cryptographic suite of the object integrity proof. | `"eddsa-jcs-2022"` | -| `object_integrity_proofs.key_id` | string | The public key ID of the object integrity proof. | `"https://example.com/actor/1#main-key"` | -| `object_integrity_proofs.signature` | string | The integrity proof of the object in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | -| `url.full` | string | The full URL being fetched by the document loader. | `"https://example.com/actor/1"` | -| `webfinger.resource` | string | The queried resource URI. | `"acct:fedify@hollo.social"` | -| `webfinger.resource.scheme` | string | The scheme of the queried resource URI. | `"acct"` | +| Attribute | Type | Description | Example | +| ---------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | +| `activitypub.activity.id` | string | The URI of the activity object. | `"https://example.com/activity/1"` | +| `activitypub.activity.type` | string[] | The qualified URI(s) of the activity type(s). | `["https://www.w3.org/ns/activitystreams#Create"]` | +| `activitypub.activity.to` | string[] | The URI(s) of the recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | +| `activitypub.activity.cc` | string[] | The URI(s) of the carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | +| `activitypub.activity.bto` | string[] | The URI(s) of the blind recipient collections/actors of the activity. | `["https://example.com/1/followers/2"]` | +| `activitypub.activity.bcc` | string[] | The URI(s) of the blind carbon-copied recipient collections/actors of the activity. | `["https://www.w3.org/ns/activitystreams#Public"]` | +| `activitypub.activity.retries` | int | The ordinal number of activity resending attempt (if and only if it's retried). | `3` | +| `activitypub.delivery.attempt` | int | The zero-based delivery attempt number for a queued outgoing activity. | `0` | +| `activitypub.delivery.permanent_failure` | boolean | Whether an outgoing delivery failure will be abandoned instead of retried. | `true` | +| `activitypub.actor.id` | string | The URI of the actor object. | `"https://example.com/actor/1"` | +| `activitypub.actor.key.cached` | boolean | Whether the actor's public keys are cached. | `true` | +| `activitypub.actor.type` | string[] | The qualified URI(s) of the actor type(s). | `["https://www.w3.org/ns/activitystreams#Person"]` | +| `activitypub.key.id` | string | The URI of the cryptographic key being verified. | `"https://example.com/actor/1#main-key"` | +| `activitypub.key_ownership.method` | string | The method used to verify key ownership (`owner_id` or `actor_fetch`). | `"actor_fetch"` | +| `activitypub.key_ownership.verified` | boolean | Whether the key ownership was successfully verified. | `true` | +| `activitypub.collection.id` | string | The URI of the collection object. | `"https://example.com/collection/1"` | +| `activitypub.collection.type` | string[] | The qualified URI(s) of the collection type(s). | `["https://www.w3.org/ns/activitystreams#OrderedCollection"]` | +| `activitypub.collection.total_items` | int | The total number of items in the collection. | `42` | +| `activitypub.object.id` | string | The URI of the object or the object enclosed by the activity. | `"https://example.com/object/1"` | +| `activitypub.object.type` | string[] | The qualified URI(s) of the object type(s). | `["https://www.w3.org/ns/activitystreams#Note"]` | +| `activitypub.object.in_reply_to` | string[] | The URI(s) of the original object to which the object reply. | `["https://example.com/object/1"]` | +| `activitypub.inboxes` | int | The number of inboxes the activity is sent to. | `12` | +| `activitypub.remote.host` | string | The hostname of the remote ActivityPub server. | `"example.com"` | +| `activitypub.shared_inbox` | boolean | Whether the activity is sent to the shared inbox. | `true` | +| `docloader.context_url` | string | The URL of the JSON-LD context document (if provided via Link header). | `"https://www.w3.org/ns/activitystreams"` | +| `docloader.document_url` | string | The final URL of the fetched document (after following redirects). | `"https://example.com/object/1"` | +| `fedify.actor.identifier` | string | The identifier of the actor. | `"1"` | +| `fedify.inbox.recipient` | string | The identifier of the inbox recipient. | `"1"` | +| `fedify.object.type` | string | The URI of the object type. | `"https://www.w3.org/ns/activitystreams#Note"` | +| `fedify.object.values.{parameter}` | string[] | The argument values of the object dispatcher. | `["1", "2"]` | +| `fedify.collection.cursor` | string | The cursor of the collection. | `"eyJpZCI6IjEiLCJ0eXBlIjoiT3JkZXJlZENvbGxlY3Rpb24ifQ=="` | +| `fedify.collection.items` | number | The number of items in the collection page. It can be less than the total items. | `10` | +| `http.redirect.url` | string | The redirect URL when a document fetch results in a redirect. | `"https://example.com/new-location"` | +| `http.response.status_code` | int | The HTTP response status code. | `200` | +| `http_signatures.signature` | string | The signature of the HTTP request in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `http_signatures.algorithm` | string | The algorithm of the HTTP request signature. | `"rsa-sha256"` | +| `http_signatures.key_id` | string | The public key ID of the HTTP request signature. | `"https://example.com/actor/1#main-key"` | +| `http_signatures.verified` | boolean | Whether the HTTP request signature was verified successfully. | `false` | +| `http_signatures.failure_reason` | string | Why HTTP signature verification failed (`noSignature`, `invalidSignature`, or `keyFetchError`). | `"keyFetchError"` | +| `http_signatures.key_fetch_status` | int | The HTTP status code from a failed signing-key fetch, when available. | `410` | +| `http_signatures.key_fetch_error` | string | The error type from a non-HTTP signing-key fetch failure, when available. | `"TypeError"` | +| `http_signatures.digest.{algorithm}` | string | The digest of the HTTP request body in hexadecimal. The `{algorithm}` is the digest algorithm (e.g., `sha`, `sha-256`). | `"d41d8cd98f00b204e9800998ecf8427e"` | +| `ld_signatures.key_id` | string | The public key ID of the Linked Data signature. | `"https://example.com/actor/1#main-key"` | +| `ld_signatures.signature` | string | The signature of the Linked Data in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `ld_signatures.type` | string | The algorithm of the Linked Data signature. | `"RsaSignature2017"` | +| `object_integrity_proofs.cryptosuite` | string | The cryptographic suite of the object integrity proof. | `"eddsa-jcs-2022"` | +| `object_integrity_proofs.key_id` | string | The public key ID of the object integrity proof. | `"https://example.com/actor/1#main-key"` | +| `object_integrity_proofs.signature` | string | The integrity proof of the object in hexadecimal. | `"73a74c990beabe6e59cc68f9c6db7811b59cbb22fd12dcffb3565b651540efe9"` | +| `url.full` | string | The full URL being fetched by the document loader. | `"https://example.com/actor/1"` | +| `webfinger.resource` | string | The queried resource URI. | `"acct:fedify@hollo.social"` | +| `webfinger.resource.scheme` | string | The scheme of the queried resource URI. | `"acct"` | [attributes]: https://opentelemetry.io/docs/specs/otel/common/#attribute [OpenTelemetry Semantic Conventions]: https://opentelemetry.io/docs/specs/semconv/ diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index 7886a28b1..e407ca2a4 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -67,10 +67,7 @@ class FederationMetrics { deliveryAttributes["activitypub.activity.type"] = activityType; } this.deliverySent.add(1, deliveryAttributes); - this.deliveryDuration.record(durationMs, { - "activitypub.remote.host": getRemoteHost(inbox), - "activitypub.delivery.success": success, - }); + this.deliveryDuration.record(durationMs, deliveryAttributes); } recordPermanentFailure(inbox: URL, statusCode: number): void { diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index 7f796aca3..47f24979c 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -554,6 +554,10 @@ test("sendActivity() records OpenTelemetry delivery metrics", async (t) => { durations[0].attributes["activitypub.remote.host"], "metrics.example", ); + assertEquals( + durations[0].attributes["activitypub.activity.type"], + "https://www.w3.org/ns/activitystreams#Create", + ); assertEquals( durations[0].attributes["activitypub.delivery.success"], true, @@ -611,6 +615,10 @@ test("sendActivity() records OpenTelemetry delivery metrics", async (t) => { durations[0].attributes["activitypub.remote.host"], "metrics.example", ); + assertEquals( + durations[0].attributes["activitypub.activity.type"], + "https://www.w3.org/ns/activitystreams#Follow", + ); assertEquals( durations[0].attributes["activitypub.delivery.success"], false, From e7249cc277549fec539012d6cd52455305677b9d Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 7 May 2026 12:41:01 +0900 Subject: [PATCH 15/16] Tighten telemetry review fixes The fixture package now declares the OpenTelemetry API dependency that its public test metric helpers expose in their exported types. The federation metric instruments also use the package metadata directly, matching the surrounding tracing code. https://github.com/fedify-dev/fedify/pull/755#pullrequestreview-4240977231 https://github.com/fedify-dev/fedify/pull/755#discussion_r3194566085 Assisted-by: gpt-5.5 --- packages/fedify/src/federation/metrics.ts | 5 +---- packages/fixture/package.json | 1 + pnpm-lock.yaml | 3 +++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/fedify/src/federation/metrics.ts b/packages/fedify/src/federation/metrics.ts index e407ca2a4..8399b375b 100644 --- a/packages/fedify/src/federation/metrics.ts +++ b/packages/fedify/src/federation/metrics.ts @@ -7,9 +7,6 @@ import { } from "@opentelemetry/api"; import metadata from "../../deno.json" with { type: "json" }; -const meterName = metadata.name || "@fedify/fedify"; -const meterVersion = metadata.version || undefined; - class FederationMetrics { readonly deliverySent: Counter; readonly deliveryPermanentFailure: Counter; @@ -18,7 +15,7 @@ class FederationMetrics { readonly inboxProcessingDuration: Histogram; constructor(meterProvider: MeterProvider) { - const meter = meterProvider.getMeter(meterName, meterVersion); + const meter = meterProvider.getMeter(metadata.name, metadata.version); this.deliverySent = meter.createCounter("activitypub.delivery.sent", { description: "ActivityPub delivery attempts.", unit: "{attempt}", diff --git a/packages/fixture/package.json b/packages/fixture/package.json index 774677bb1..aecaaf131 100644 --- a/packages/fixture/package.json +++ b/packages/fixture/package.json @@ -34,6 +34,7 @@ "dependencies": { "@fedify/vocab-runtime": "workspace:^", "@logtape/logtape": "catalog:", + "@opentelemetry/api": "catalog:", "@opentelemetry/core": "catalog:", "@opentelemetry/sdk-trace-base": "catalog:" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0253d9b6..9b27433ed 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1243,6 +1243,9 @@ importers: '@logtape/logtape': specifier: 'catalog:' version: 2.0.5 + '@opentelemetry/api': + specifier: 'catalog:' + version: 1.9.0 '@opentelemetry/core': specifier: 'catalog:' version: 2.5.0(@opentelemetry/api@1.9.0) From f386a47b4fa16ee8a8526840924a31ea97a9f6c4 Mon Sep 17 00:00:00 2001 From: Hong Minhee Date: Thu, 7 May 2026 13:16:35 +0900 Subject: [PATCH 16/16] Preserve outbound trace activity details Outbound send activity events now keep the lightweight activity type and actor identifiers that the debug exporter needs after the full activity JSON payload was removed from the event attributes. https://github.com/fedify-dev/fedify/pull/755#discussion_r3198889460 Assisted-by: gpt-5.5 --- docs/manual/opentelemetry.md | 19 ++++++++--- packages/fedify/src/federation/send.test.ts | 8 +++++ packages/fedify/src/federation/send.ts | 36 +++++++++++++++++++-- packages/fedify/src/otel/exporter.test.ts | 15 ++++++++- packages/fedify/src/otel/exporter.ts | 11 +++++-- 5 files changed, 80 insertions(+), 9 deletions(-) diff --git a/docs/manual/opentelemetry.md b/docs/manual/opentelemetry.md index 14892453c..ff81f27dc 100644 --- a/docs/manual/opentelemetry.md +++ b/docs/manual/opentelemetry.md @@ -265,11 +265,14 @@ Each span event includes attributes with detailed information: - `activitypub.inbox.url`: The inbox URL where the activity was delivered - `activitypub.activity.id`: The activity ID + - `activitypub.activity.type` (optional): The qualified activity type URI + - `activitypub.actor.id` (optional): The sender actor ID -The `activitypub.activity.sent` event records delivery metadata only. It does -not include the full `activitypub.activity.json` payload; if you need the full -outbound activity for auditing, store it in your application before delivery -and correlate it with `activitypub.activity.id`. +The `activitypub.activity.sent` event records delivery metadata and lightweight +activity identifiers only. It does not include the full +`activitypub.activity.json` payload; if you need the full outbound activity for +auditing, store it in your application before delivery and correlate it with +`activitypub.activity.id`. **`activitypub.delivery.failed` event attributes:** @@ -462,9 +465,17 @@ export class FedifyDebugExporter implements SpanExporter { const inboxUrl = event.attributes[ "activitypub.inbox.url" ] as string | undefined; + const activityType = event.attributes[ + "activitypub.activity.type" + ] as string | undefined; + const actorId = event.attributes[ + "activitypub.actor.id" + ] as string | undefined; this.activities.push({ direction: "outbound", activityId, + activityType, + actorId, inboxUrl, timestamp: new Date(span.startTime[0] * 1000), }); diff --git a/packages/fedify/src/federation/send.test.ts b/packages/fedify/src/federation/send.test.ts index 47f24979c..f6ae684d1 100644 --- a/packages/fedify/src/federation/send.test.ts +++ b/packages/fedify/src/federation/send.test.ts @@ -498,6 +498,14 @@ test("sendActivity() records OpenTelemetry span events", async (t) => { event.attributes["activitypub.activity.id"], "https://example.com/activity", ); + assertEquals( + event.attributes["activitypub.activity.type"], + "https://www.w3.org/ns/activitystreams#Create", + ); + assertEquals( + event.attributes["activitypub.actor.id"], + "https://example.com/person", + ); assertEquals(event.attributes["activitypub.activity.json"], undefined); exporter.clear(); diff --git a/packages/fedify/src/federation/send.ts b/packages/fedify/src/federation/send.ts index ea847be6a..263af1de1 100644 --- a/packages/fedify/src/federation/send.ts +++ b/packages/fedify/src/federation/send.ts @@ -1,6 +1,7 @@ import type { Recipient } from "@fedify/vocab"; import { getLogger } from "@logtape/logtape"; import { + type Attributes, type MeterProvider, type Span, SpanKind, @@ -198,6 +199,29 @@ export function sendActivity( const MAX_ERROR_RESPONSE_BODY_BYTES = 1024; +function getActivityActorId(activity: unknown): string | undefined { + if (!isRecord(activity)) return undefined; + return getIdValue(activity.actor); +} + +function getIdValue(value: unknown): string | undefined { + if (typeof value === "string" && value !== "") return value; + if (value instanceof URL) return value.href; + if (Array.isArray(value)) { + for (const item of value) { + const id = getIdValue(item); + if (id != null) return id; + } + return undefined; + } + if (isRecord(value)) return getIdValue(value.id); + return undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value != null; +} + async function readLimitedResponseBody( response: Response, maxBytes: number, @@ -340,10 +364,18 @@ async function sendActivityInternal( deliverySuccess = true; // Record the sent activity with delivery details - span.addEvent("activitypub.activity.sent", { + const eventAttributes: Attributes = { "activitypub.inbox.url": inbox.href, "activitypub.activity.id": activityId ?? "", - }); + }; + if (activityType != null) { + eventAttributes["activitypub.activity.type"] = activityType; + } + const actorId = getActivityActorId(activity); + if (actorId != null) { + eventAttributes["activitypub.actor.id"] = actorId; + } + span.addEvent("activitypub.activity.sent", eventAttributes); } finally { federationMetrics.recordDelivery( inbox, diff --git a/packages/fedify/src/otel/exporter.test.ts b/packages/fedify/src/otel/exporter.test.ts index 58e1bf710..d5ebd7cdf 100644 --- a/packages/fedify/src/otel/exporter.test.ts +++ b/packages/fedify/src/otel/exporter.test.ts @@ -99,11 +99,19 @@ function createActivitySentEvent(options: { activityJson?: string; inboxUrl: string; activityId?: string; + activityType?: string; + actorId?: string; }): TimedEvent { const attributes: Attributes = { "activitypub.inbox.url": options.inboxUrl, "activitypub.activity.id": options.activityId ?? "", }; + if (options.activityType != null) { + attributes["activitypub.activity.type"] = options.activityType; + } + if (options.actorId != null) { + attributes["activitypub.actor.id"] = options.actorId; + } if (options.activityJson != null) { attributes["activitypub.activity.json"] = options.activityJson; } @@ -180,6 +188,8 @@ test("FedifySpanExporter", async (t) => { const spanId = "span012"; const inboxUrl = "https://example.com/users/alice/inbox"; const activityId = "https://myserver.com/activities/789"; + const activityType = "https://www.w3.org/ns/activitystreams#Accept"; + const actorId = "https://myserver.com/users/bob"; const span = createMockSpan({ traceId, @@ -189,6 +199,8 @@ test("FedifySpanExporter", async (t) => { createActivitySentEvent({ inboxUrl, activityId, + activityType, + actorId, }), ], }); @@ -205,8 +217,9 @@ test("FedifySpanExporter", async (t) => { assertEquals(activities[0].traceId, traceId); assertEquals(activities[0].spanId, spanId); assertEquals(activities[0].direction, "outbound"); - assertEquals(activities[0].activityType, "Unknown"); + assertEquals(activities[0].activityType, activityType); assertEquals(activities[0].activityId, activityId); + assertEquals(activities[0].actorId, actorId); assertEquals(activities[0].activityJson, undefined); assertEquals(activities[0].inboxUrl, inboxUrl); }, diff --git a/packages/fedify/src/otel/exporter.ts b/packages/fedify/src/otel/exporter.ts index 8c87692d3..af7f0edf5 100644 --- a/packages/fedify/src/otel/exporter.ts +++ b/packages/fedify/src/otel/exporter.ts @@ -432,18 +432,25 @@ export class FedifySpanExporter implements SpanExporter { const inboxUrl = attrs["activitypub.inbox.url"]; const explicitActivityId = attrs["activitypub.activity.id"]; + const explicitActivityType = attrs["activitypub.activity.type"]; + const explicitActorId = attrs["activitypub.actor.id"]; return { traceId, spanId, parentSpanId, direction: "outbound", - activityType, + activityType: + typeof explicitActivityType === "string" && explicitActivityType !== "" + ? explicitActivityType + : activityType, activityId: activityId ?? (typeof explicitActivityId === "string" && explicitActivityId !== "" ? explicitActivityId : undefined), - actorId, + actorId: typeof explicitActorId === "string" && explicitActorId !== "" + ? explicitActorId + : actorId, ...(typeof activityJson === "string" ? { activityJson } : {}), timestamp: new Date( event.time[0] * 1000 + event.time[1] / 1e6,