feat: adds a standalone transaction-clearer bot for recovering from nonce backlogs.

packages/monitor-v2/src/bot-oo/index.ts

@@ -1,8 +1,138 @@
 import { delay, waitForLogger, GasEstimator } from "@uma/financial-templates-lib";
-import { BotModes, initMonitoringParams, Logger, startupLogLevel } from "./common";
+import { BigNumber } from "ethers";
+import { BotModes, MonitoringParams, initMonitoringParams, Logger, startupLogLevel } from "./common";
 import { settleRequests } from "./SettleRequests";
 
 const logger = Logger;
+const DEFAULT_REPLACEMENT_BUMP_PERCENT = 20;
+const DEFAULT_REPLACEMENT_ATTEMPTS = 3;
+
+type NonceBacklogConfig = {
+  replacementBumpPercent: number;
+  replacementAttempts: number;
+};
+
+const parsePositiveInt = (value: string | undefined, defaultValue: number, name: string): number => {
+  if (value === undefined) return defaultValue;
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0 || !Number.isInteger(parsed)) {
+    throw new Error(`${name} must be a positive integer, got: ${value}`);
+  }
+  return parsed;
+};
+
+const getNonceBacklogConfig = (env: NodeJS.ProcessEnv): NonceBacklogConfig => ({
+  replacementBumpPercent: parsePositiveInt(
+    env.NONCE_REPLACEMENT_BUMP_PERCENT,
+    DEFAULT_REPLACEMENT_BUMP_PERCENT,
+    "NONCE_REPLACEMENT_BUMP_PERCENT"
+  ),
+  replacementAttempts: parsePositiveInt(
+    env.NONCE_REPLACEMENT_ATTEMPTS,
+    DEFAULT_REPLACEMENT_ATTEMPTS,
+    "NONCE_REPLACEMENT_ATTEMPTS"
+  ),
+});
+
+function bumpFeeData(
+  feeData: ReturnType<GasEstimator["getCurrentFastPriceEthers"]>,
+  bumps: number,
+  config: NonceBacklogConfig
+): ReturnType<GasEstimator["getCurrentFastPriceEthers"]> {
+  if (bumps === 0) return feeData;
+
+  const bumpValue = (value: BigNumber) => {
+    let bumped = value;
+    for (let i = 0; i < bumps; i++) {
+      bumped = bumped.mul(100 + config.replacementBumpPercent).div(100);
+    }
+    return bumped;
+  };
+
+  if ("gasPrice" in feeData) {
+    return { gasPrice: bumpValue(feeData.gasPrice) };
+  }
+
+  return {
+    maxFeePerGas: bumpValue(feeData.maxFeePerGas),
+    maxPriorityFeePerGas: bumpValue(feeData.maxPriorityFeePerGas),
+  };
+}
+
+async function handleNonceBacklog(
+  params: MonitoringParams,
+  gasEstimator: GasEstimator,
+  config: NonceBacklogConfig
+): Promise<boolean> {
+  const botAddress = await params.signer.getAddress();
+  const [latestNonce, pendingNonce] = await Promise.all([
+    params.provider.getTransactionCount(botAddress, "latest"),
+    params.provider.getTransactionCount(botAddress, "pending"),
+  ]);
+
+  if (pendingNonce <= latestNonce) return false;
+
+  logger.warn({
+    at: "OracleBot",
+    message: "Nonce backlog detected, skipping settlements for this run",
+    botAddress,
+    latestNonce,
+    pendingNonce,
+  });
+
+  await gasEstimator.update();
+  const baseFeeData = gasEstimator.getCurrentFastPriceEthers();
+
+  for (let attempt = 1; attempt <= config.replacementAttempts; attempt++) {
+    const feeData = bumpFeeData(baseFeeData, attempt - 1, config);
+    try {
+      const tx = await params.signer.sendTransaction({
+        to: botAddress,
+        value: 0,
+        nonce: latestNonce,
+        gasLimit: 21_000,
+        ...feeData,
+      });
+
+      logger.info({
+        at: "OracleBot",
+        message: "Submitted nonce backlog cancellation transaction",
+        tx: tx.hash,
+        nonce: latestNonce,
+        attempt,
+        feeData,
+      });
+
+      await tx.wait(1);
+
+      logger.info({
+        at: "OracleBot",
+        message: "Nonce backlog cancellation transaction mined",
+        tx: tx.hash,
+        nonce: latestNonce,
+        attempt,
+      });
+      return true;
+    } catch (error) {
+      logger.warn({
+        at: "OracleBot",
+        message: "Nonce backlog cancellation transaction failed",
+        attempt,
+        nonce: latestNonce,
+        error,
+      });
+    }
+  }
+
+  logger.error({
+    at: "OracleBot",
+    message: "Failed to clear nonce backlog, exiting early",
+    nonce: latestNonce,
+    pendingNonce,
+  });
+
+  return true;
+}
 
 async function main() {
   const params = await initMonitoringParams(process.env);
@@ -16,12 +146,20 @@ async function main() {
   });
 
   const gasEstimator = new GasEstimator(logger, undefined, params.chainId, params.provider);
+  const nonceBacklogConfig = getNonceBacklogConfig(process.env);
 
   const cmds = {
     settleRequestsEnabled: settleRequests,
   };
 
   for (;;) {
+    const backlogDetected = await handleNonceBacklog(params, gasEstimator, nonceBacklogConfig);
+    if (backlogDetected) {
+      await delay(5); // Let any in-flight logs flush before exiting.
+      await waitForLogger(logger);
+      break;
+    }
+
     await gasEstimator.update();
 
     const runCmds = Object.entries(cmds)

packages/monitor-v2/src/transaction-clearer/TransactionClearer.ts

@@ -0,0 +1,171 @@
+import { BigNumber } from "ethers";
+import type { Logger as LoggerType } from "winston";
+import type { Provider } from "@ethersproject/abstract-provider";
+import { GasEstimator } from "@uma/financial-templates-lib";
+import { MonitoringParams, NonceBacklogConfig } from "./common";
+
+type FeeData = { maxFeePerGas: BigNumber; maxPriorityFeePerGas: BigNumber } | { gasPrice: BigNumber };
+
+function isLondonFeeData(feeData: FeeData): feeData is { maxFeePerGas: BigNumber; maxPriorityFeePerGas: BigNumber } {
+  return "maxFeePerGas" in feeData;
+}
+
+function bumpFeeData(baseFeeData: FeeData, attemptIndex: number, config: NonceBacklogConfig): FeeData {
+  // Calculate multiplier: (numerator/denominator)^(attemptIndex+1)
+  // For attempt 0: 1.2x, attempt 1: 1.44x, attempt 2: 1.73x (with default 12/10)
+  let numerator = BigNumber.from(config.feeBumpNumerator);
+  let denominator = BigNumber.from(config.feeBumpDenominator);
+
+  for (let i = 0; i < attemptIndex; i++) {
+    numerator = numerator.mul(config.feeBumpNumerator);
+    denominator = denominator.mul(config.feeBumpDenominator);
+  }
+
+  if (isLondonFeeData(baseFeeData)) {
+    return {
+      maxFeePerGas: baseFeeData.maxFeePerGas.mul(numerator).div(denominator),
+      maxPriorityFeePerGas: baseFeeData.maxPriorityFeePerGas.mul(numerator).div(denominator),
+    };
+  } else {
+    return {
+      gasPrice: baseFeeData.gasPrice.mul(numerator).div(denominator),
+    };
+  }
+}
+
+async function getNonces(
+  provider: Provider,
+  address: string
+): Promise<{ latestNonce: number; pendingNonce: number }> {
+  const [latestNonce, pendingNonce] = await Promise.all([
+    provider.getTransactionCount(address, "latest"),
+    provider.getTransactionCount(address, "pending"),
+  ]);
+  return { latestNonce, pendingNonce };
+}
+
+export async function clearStuckTransactions(
+  logger: LoggerType,
+  params: MonitoringParams,
+  gasEstimator: GasEstimator
+): Promise<void> {
+  const { provider, signer, nonceBacklogConfig } = params;
+  const botAddress = await signer.getAddress();
+
+  const { latestNonce, pendingNonce } = await getNonces(provider, botAddress);
+  const backlog = pendingNonce - latestNonce;
+
+  if (backlog < nonceBacklogConfig.nonceBacklogThreshold) {
+    logger.debug({
+      at: "TransactionClearer",
+      message: "No nonce backlog detected",
+      botAddress,
+      latestNonce,
+      pendingNonce,
+      backlog,
+      threshold: nonceBacklogConfig.nonceBacklogThreshold,
+    });
+    return;
+  }
+
+  logger.warn({
+    at: "TransactionClearer",
+    message: "Nonce backlog detected, attempting to clear stuck transactions",
+    botAddress,
+    latestNonce,
+    pendingNonce,
+    backlog,
+    threshold: nonceBacklogConfig.nonceBacklogThreshold,
+  });
+
+  // Get base fee data from gas estimator
+  const baseFeeData = gasEstimator.getCurrentFastPriceEthers();
+
+  // Clear all stuck nonces from latestNonce to pendingNonce - 1
+  for (let nonce = latestNonce; nonce < pendingNonce; nonce++) {
+    let cleared = false;
+
+    for (let attempt = 0; attempt < nonceBacklogConfig.replacementAttempts; attempt++) {
+      const feeData = bumpFeeData(baseFeeData, attempt, nonceBacklogConfig);
+
+      try {
+        logger.info({
+          at: "TransactionClearer",
+          message: `Attempting to clear stuck transaction (nonce ${nonce}, attempt ${attempt + 1})`,
+          botAddress,
+          nonce,
+          attempt: attempt + 1,
+          feeData: isLondonFeeData(feeData)
+            ? {
+                maxFeePerGas: feeData.maxFeePerGas.toString(),
+                maxPriorityFeePerGas: feeData.maxPriorityFeePerGas.toString(),
+              }
+            : { gasPrice: feeData.gasPrice.toString() },
+        });
+
+        const tx = await signer.sendTransaction({
+          to: botAddress, // Self-transaction
+          value: 0,
+          nonce,
+          gasLimit: 21_000,
+          ...feeData,
+        });
+
+        const receipt = await tx.wait(1);
+
+        logger.info({
+          at: "TransactionClearer",
+          message: `Successfully cleared stuck transaction (nonce ${nonce})`,
+          botAddress,
+          nonce,
+          transactionHash: receipt.transactionHash,
+          gasUsed: receipt.gasUsed.toString(),
+        });
+
+        cleared = true;
+        break;
+      } catch (error) {
+        logger.warn({
+          at: "TransactionClearer",
+          message: `Failed to clear stuck transaction (nonce ${nonce}, attempt ${attempt + 1})`,
+          botAddress,
+          nonce,
+          attempt: attempt + 1,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    }
+
+    if (!cleared) {
+      logger.error({
+        at: "TransactionClearer",
+        message: `Failed to clear stuck transaction after all attempts (nonce ${nonce})`,
+        botAddress,
+        nonce,
+        maxAttempts: nonceBacklogConfig.replacementAttempts,
+      });
+    }
+  }
+
+  // Verify final state
+  const { latestNonce: finalLatestNonce, pendingNonce: finalPendingNonce } = await getNonces(provider, botAddress);
+  const finalBacklog = finalPendingNonce - finalLatestNonce;
+
+  if (finalBacklog < nonceBacklogConfig.nonceBacklogThreshold) {
+    logger.info({
+      at: "TransactionClearer",
+      message: "Successfully cleared nonce backlog",
+      botAddress,
+      previousBacklog: backlog,
+      finalBacklog,
+    });
+  } else {
+    logger.warn({
+      at: "TransactionClearer",
+      message: "Nonce backlog still present after clearing attempt",
+      botAddress,
+      previousBacklog: backlog,
+      finalBacklog,
+    });
+  }
+}

packages/monitor-v2/src/transaction-clearer/common.ts

@@ -0,0 +1,34 @@
+export { Logger } from "@uma/financial-templates-lib";
+import { BaseMonitoringParams, initBaseMonitoringParams, startupLogLevel as baseStartup } from "../bot-utils/base";
+
+export interface NonceBacklogConfig {
+  // Minimum nonce difference (pending - latest) to trigger clearing
+  nonceBacklogThreshold: number;
+  // Fee bump multiplier: bumpedFee = fee * numerator / denominator
+  feeBumpNumerator: number;
+  feeBumpDenominator: number;
+  // Max attempts to replace a stuck transaction with increasing fees
+  replacementAttempts: number;
+}
+
+export interface MonitoringParams extends BaseMonitoringParams {
+  nonceBacklogConfig: NonceBacklogConfig;
+}
+
+export const initMonitoringParams = async (env: NodeJS.ProcessEnv): Promise<MonitoringParams> => {
+  const base = await initBaseMonitoringParams(env);
+
+  const nonceBacklogConfig: NonceBacklogConfig = {
+    nonceBacklogThreshold: Number(env.NONCE_BACKLOG_THRESHOLD) || 1,
+    feeBumpNumerator: Number(env.FEE_BUMP_NUMERATOR) || 12,
+    feeBumpDenominator: Number(env.FEE_BUMP_DENOMINATOR) || 10,
+    replacementAttempts: Number(env.REPLACEMENT_ATTEMPTS) || 3,
+  };
+
+  return {
+    ...base,
+    nonceBacklogConfig,
+  };
+};
+
+export const startupLogLevel = baseStartup;