feat: instrument command latency in Sentry

apps/api/package.json

@@ -20,6 +20,7 @@
     "@nestjs/platform-fastify": "^11.1.6",
     "@nestjs/swagger": "^11.2.0",
     "@sentry/node": "^7.118.0",
+    "@sentry/profiling-node": "^7.120.4",
     "@statsify/api-client": "workspace:^",
     "@statsify/assets": "workspace:^",
     "@statsify/logger": "workspace:^",

apps/api/src/hypixel/hypixel.service.ts

@@ -145,8 +145,12 @@ export class HypixelService {
     const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
 
     const child = transaction?.startChild({
-      op: "http.client",
+      op: "hypixel.api.fetch",
       description: `GET ${this.httpService.axiosRef.getUri({ url })}`,
+      data: {
+        "http.method": "GET",
+        "http.route": url,
+      },
     });
 
     return this.httpService.get(url, { params }).pipe(
@@ -155,14 +159,16 @@ export class HypixelService {
         child?.finish();
       }),
       map((res) => res.data),
-      catchError((err) =>
-        throwError(
+      catchError((err) => {
+        child?.finish();
+
+        return throwError(
           () =>
             new Error(`Fetching ${url} failed with reason: ${err.message}`, {
               cause: err,
             })
-        )
-      )
+        );
+      })
     );
   }
 }

apps/api/src/index.ts

@@ -21,6 +21,7 @@ import { config } from "@statsify/util";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { mkdir } from "node:fs/promises";
+import { nodeProfilingIntegration } from "@sentry/profiling-node";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -31,16 +32,23 @@ process.on("uncaughtException", handleError);
 process.on("unhandledRejection", handleError);
 
 const sentryDsn = await config("sentry.apiDsn", { required: false });
+const sentryTracesSampleRate =
+  await config("sentry.tracesSampleRate", { required: false }) ?? 0;
+const sentryProfilesSampleRate =
+  await config("sentry.profilesSampleRate", { required: false }) ??
+  sentryTracesSampleRate;
 
 if (sentryDsn) {
   Sentry.init({
     dsn: sentryDsn,
     integrations: [
       new Sentry.Integrations.Http({ tracing: false, breadcrumbs: true }),
       new Sentry.Integrations.Mongo({ useMongoose: true }),
+      nodeProfilingIntegration(),
     ],
     normalizeDepth: 3,
-    tracesSampleRate: await config("sentry.tracesSampleRate"),
+    tracesSampleRate: sentryTracesSampleRate,
+    profilesSampleRate: sentryProfilesSampleRate,
     environment: await config("environment"),
   });
 }

apps/api/src/redis/redis.utils.ts

@@ -6,6 +6,7 @@
  * https://github.com/Statsify/statsify/blob/main/LICENSE
  */
 
+import * as Sentry from "@sentry/node";
 import {
   REDIS_MODULE_CONNECTION,
   REDIS_MODULE_CONNECTION_TOKEN,
@@ -24,5 +25,20 @@ export function getRedisConnectionToken(connection?: string): string {
 
 export function createRedisConnection(options: RedisModuleOptions) {
   const { config } = options;
-  return config.url ? new Redis(config.url, config) : new Redis(config);
+  const redis = config.url ? new Redis(config.url, config) : new Redis(config);
+  const sendCommand = redis.sendCommand.bind(redis);
+
+  redis.sendCommand = ((command, stream) => {
+    const commandName = String((command as { name: string }).name);
+    const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
+    const span = transaction?.startChild({
+      op: "redis.query",
+      description: commandName,
+      data: { "redis.command": commandName },
+    });
+
+    return (sendCommand(command, stream) as Promise<unknown>).finally(() => span?.finish());
+  }) as Redis["sendCommand"];
+
+  return redis;
 }

apps/discord-bot/package.json

@@ -12,6 +12,7 @@
   },
   "dependencies": {
     "@sentry/node": "^7.118.0",
+    "@sentry/profiling-node": "^7.120.4",
     "@statsify/api-client": "workspace:^",
     "@statsify/assets": "workspace:^",
     "@statsify/discord": "workspace:^",

apps/discord-bot/src/commands/base.hypixel-command.ts

@@ -59,6 +59,7 @@ export interface BaseHypixelCommand<T extends GamesWithBackgrounds, K = never> {
   description: "",
   args: [PlayerArgument],
   cooldown: 10,
+  group: "hypixel",
 })
 export abstract class BaseHypixelCommand<T extends GamesWithBackgrounds, K = never> {
   protected readonly apiService: ApiService;

apps/discord-bot/src/commands/ratios/ratios.command.tsx

@@ -6,6 +6,7 @@
  * https://github.com/Statsify/statsify/blob/main/LICENSE
  */
 
+import * as Sentry from "@sentry/node";
 import {
   ARCADE_MODES,
   ARENA_BRAWL_MODES,
@@ -57,7 +58,7 @@ import { render } from "@statsify/rendering";
 
 const args = [PlayerArgument];
 
-@Command({ description: (t) => t("commands.ratios") })
+@Command({ description: (t) => t("commands.ratios"), group: "hypixel" })
 export class RatiosCommand {
   public constructor(
     private readonly apiService: ApiService,
@@ -235,7 +236,20 @@ export class RatiosCommand {
         };
 
         const canvas = render(<RatiosProfile {...props} />, getTheme(user));
-        const buffer = await canvas.toBuffer("png");
+        const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
+        const span = transaction?.startChild({
+          op: "canvas.encode_png",
+          description: "Encode ratios canvas as PNG",
+        });
+
+        let buffer: Buffer;
+
+        try {
+          buffer = await canvas.toBuffer("png");
+          span?.setData("png.bytes", buffer.byteLength);
+        } finally {
+          span?.finish();
+        }
 
         return {
           files: [{ name: "ratios.png", data: buffer, type: "image/png" }],

apps/discord-bot/src/index.ts

@@ -17,6 +17,7 @@ import { VerifyCommand } from "#commands/verify.command";
 import { config } from "@statsify/util";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { nodeProfilingIntegration } from "@sentry/profiling-node";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
@@ -27,13 +28,22 @@ process.on("uncaughtException", handleError);
 process.on("unhandledRejection", handleError);
 
 const sentryDsn = await config("sentry.discordBotDsn", { required: false });
+const sentryTracesSampleRate =
+  await config("sentry.tracesSampleRate", { required: false }) ?? 0;
+const sentryProfilesSampleRate =
+  await config("sentry.profilesSampleRate", { required: false }) ??
+  sentryTracesSampleRate;
 
 if (sentryDsn) {
   Sentry.init({
     dsn: sentryDsn,
-    integrations: [new Sentry.Integrations.Http({ tracing: false, breadcrumbs: true })],
+    integrations: [
+      new Sentry.Integrations.Http({ tracing: false, breadcrumbs: true }),
+      nodeProfilingIntegration(),
+    ],
     normalizeDepth: 3,
-    tracesSampleRate: await config("sentry.tracesSampleRate"),
+    tracesSampleRate: sentryTracesSampleRate,
+    profilesSampleRate: sentryProfilesSampleRate,
     environment: await config("environment"),
   });
 }

apps/discord-bot/src/lib/command.listener.ts

@@ -65,12 +65,31 @@ export class CommandListener extends AbstractCommandListener {
       parentData
     );
 
-    const transaction = Sentry.startTransaction({ name: commandName, op: "command" });
+    const [name, ...subcommandParts] = commandName.split(" ");
+    const subcommand = subcommandParts.length ? commandName : undefined;
+
+    const transaction = Sentry.startTransaction({
+      name: commandName,
+      op: "discord.command.total",
+      data: {
+        "command.name": name,
+        "command.group": parentCommand.group ?? command.group ?? "unknown",
+        "command.subcommand": subcommand,
+      },
+      tags: {
+        "command.name": name,
+        "command.group": parentCommand.group ?? command.group ?? "unknown",
+        "command.subcommand": subcommand ?? "none",
+      },
+    });
 
     Sentry.configureScope((scope) => scope.setSpan(transaction));
 
     Sentry.setContext("command", {
       command: commandName,
+      group: parentCommand.group ?? command.group ?? null,
+      name,
+      subcommand: subcommand ?? null,
       options: data.options,
       guild: interaction.getGuildId() ?? null,
     });

packages/api-client/src/api.service.ts

@@ -7,7 +7,13 @@
  */
 
 import * as Sentry from "@sentry/node";
-import Axios, { AxiosInstance, AxiosRequestHeaders, Method, ResponseType } from "axios";
+import Axios, {
+  AxiosInstance,
+  AxiosRequestHeaders,
+  AxiosResponse,
+  Method,
+  ResponseType,
+} from "axios";
 import {
   CacheLevel,
   GuildQuery,
@@ -291,21 +297,30 @@ export class ApiService {
     const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
 
     const child = transaction?.startChild({
-      op: "http.client",
+      op: "statsify.api.fetch",
       description: `${method} ${url}`,
+      data: {
+        "http.method": method,
+        "http.route": url,
+      },
     });
 
-    const res = await this.axios.request({
-      url,
-      method,
-      params,
-      headers,
-      data: body,
-      responseType,
-    });
-
-    child?.setHttpStatus(res.status);
-    child?.finish();
+    let res: AxiosResponse<any>;
+
+    try {
+      res = await this.axios.request({
+        url,
+        method,
+        params,
+        headers,
+        data: body,
+        responseType,
+      });
+
+      child?.setHttpStatus(res.status);
+    } finally {
+      child?.finish();
+    }
 
     const data = res.data;
 

packages/discord/src/command/abstract-command.listener.ts

@@ -130,27 +130,43 @@ export abstract class AbstractCommandListener {
     message,
   }: ExecuteCommandOptions) {
     const transaction = Sentry.getCurrentHub().getScope()?.getTransaction();
+    const commandSpan = transaction?.startChild({
+      op: "discord.command.execute",
+      description: commandName,
+    });
 
     try {
       preconditions.forEach((precondition) => precondition());
 
       const response = await command.execute(context);
+      commandSpan?.finish();
 
-      if (typeof response !== "object") return;
-      transaction?.finish();
+      if (typeof response !== "object") {
+        this.setMemoryUsage(transaction);
+        transaction?.finish();
+        return;
+      }
 
-      context.reply({
+      await context.reply({
         ...message,
         ...response,
       });
+
+      this.setMemoryUsage(transaction);
+      transaction?.finish();
     } catch (err) {
       if (err instanceof Message) {
+        commandSpan?.finish();
+        await context.reply(err);
+        this.setMemoryUsage(transaction);
         transaction?.finish();
-        return context.reply(err);
+        return;
       }
 
       this.logger.error(`An error occurred when running "${commandName}"`);
       this.logger.error(err);
+      commandSpan?.finish();
+      this.setMemoryUsage(transaction);
       transaction?.finish();
     }
   }
@@ -318,5 +334,14 @@ export abstract class AbstractCommandListener {
     });
   }
 
+  private setMemoryUsage(transaction?: Sentry.Transaction) {
+    if (!transaction) return;
+
+    const { rss, heapUsed } = process.memoryUsage();
+
+    transaction.setData("memory.rss.bytes", rss);
+    transaction.setData("memory.heap_used.bytes", heapUsed);
+  }
+
   protected abstract onCommand(interaction: Interaction): Promise<void> | void;
 }

packages/discord/src/command/command.interface.ts

@@ -27,6 +27,11 @@ export interface CommandOptions {
 
   cooldown?: number;
 
+  /**
+   * The product area this command belongs to. Used for observability.
+   */
+  group?: string;
+
   /**
    * The minimum user tier required to use this command.
    */

packages/discord/src/command/command.resolvable.ts

@@ -33,6 +33,7 @@ export class CommandResolvable {
 
   public args: AbstractArgument[];
   public cooldown: number;
+  public group?: string;
   public tier: UserTier;
   public preview?: string;
 
@@ -49,6 +50,7 @@ export class CommandResolvable {
       methodName,
       tier = UserTier.NONE,
       preview,
+      group,
       cooldown = 10,
     }: CommandMetadata,
     target: any
@@ -72,6 +74,7 @@ export class CommandResolvable {
 
     this.type = ApplicationCommandType.ChatInput;
     this.cooldown = cooldown;
+    this.group = group;
 
     const argsResolved = (args ?? [])?.map((a) =>
       a instanceof AbstractArgument ? a : new a()