fix(billing): migrate the ProcessedStripeEvent TTL index to its dead-letter-preserving spec

modules/billing/migrations/20260729120000-fix-processed-stripe-event-ttl-index-partial-filter.js

@@ -0,0 +1,193 @@
+/**
+ * Module dependencies
+ */
+import mongoose from 'mongoose';
+
+const INDEX_NAME = 'processedAt_1';
+const INDEX_KEY = { processedAt: 1 };
+const TTL_SECONDS = 30 * 24 * 60 * 60;
+
+/**
+ * @desc Exact-key match helper: a single-field index keyed (processedAt:1).
+ * @param {Object} ix - an index document from listIndexes()
+ * @returns {boolean} true when the index key is exactly { processedAt: 1 }
+ */
+const sameKey = (ix) => {
+  const keys = Object.keys(ix.key || {});
+  return keys.length === 1 && keys[0] === 'processedAt' && ix.key.processedAt === 1;
+};
+
+/**
+ * @desc Exact-spec match helper: `sameKey` PLUS the correct name, TTL, and
+ * partialFilterExpression — the full target shape this migration installs.
+ * @param {Object} ix - an index document from listIndexes()
+ * @returns {boolean} true when ix is already the fully-installed target index
+ */
+const isExactTargetIndex = (ix) =>
+  sameKey(ix) && ix.name === INDEX_NAME && ix.expireAfterSeconds === TTL_SECONDS && ix.partialFilterExpression?.deadLetter?.$eq === false;
+
+/**
+ * Migration: dead-letter-preserving ProcessedStripeEvent TTL index (#4006).
+ *
+ * The schema's 30-day TTL index on `processedAt` gained
+ * `partialFilterExpression: { deadLetter: { $eq: false } }` (dead-letter
+ * documents are permanent rejections — Stripe was told to stop retrying —
+ * and must NEVER be purged, or a later manual replay would re-process the
+ * event as if new: double credit grants, double subscription resets) with NO
+ * accompanying migration. On an already-deployed database still holding the
+ * plain `processedAt_1` TTL index, autoIndex therefore hits
+ * IndexOptionsConflict (code 85: same name, different options) on every
+ * boot. Since #4004, `awaitIndexBuilds()` tolerates that conflict (logs
+ * loudly, keeps serving on the stale live index) precisely so THIS migration
+ * — the authoritative repair — can run right after in bootstrap. Without a
+ * migration the tolerated conflict would persist forever, with the live
+ * index still purging dead-letter documents after 30 days.
+ *
+ * Second latent defect fixed here (backfill): documents written before the
+ * `deadLetter` field existed carry no `deadLetter` at all. A
+ * partialFilterExpression uses query match semantics, and
+ * `{ deadLetter: { $eq: false } }` does NOT match a missing field — such
+ * documents would fall outside the new TTL entirely and never expire.
+ * Backfilling `deadLetter: false` restores them into the TTL's scope; the
+ * schema default covers every new write. Why the backfill is safe: `attempts`
+ * / `lastError` / `lastErrorAt` / `deadLetter` all arrived together with the
+ * dead-letter mechanism itself, and the schema default has materialized
+ * `deadLetter: false` on every document written since — so a document missing
+ * `deadLetter` predates the mechanism entirely (a plain success claim) and
+ * cannot be a dead-lettered rejection. That reasoning holds for every code
+ * path but not for manual surgery (hand edits, partial restores), so instead
+ * of trusting it blindly the pre-check below ABORTS LOUD on the
+ * impossible-by-code state: a document missing `deadLetter` that nevertheless
+ * carries dead-letter-era evidence (`attempts > 0` or a non-null
+ * `lastError`). Flipping such a document to `false` on a guess could mark a
+ * genuine permanent rejection as purgeable — the TTL would expire it and a
+ * later Stripe-dashboard replay would re-process the event as new (double
+ * credit grant / double subscription reset). An operator classifies those by
+ * hand; this migration will NOT pick.
+ *
+ * Same-name swap, deliberately NOT the distinct-name technique of
+ * `20260728120000-fix-usage-weekkey-index-partial.js`: the schema already
+ * ships this spec under the default name (`processedAt_1`), so renaming now
+ * would churn every already-converged database (drop a correct index to
+ * install an identical twin under a new name) just to spare legacy databases
+ * a conflict that #4004 already made survivable. On a legacy database this
+ * migration is REACHABLE only because of #4004 — the boot-tolerance fix must
+ * be deployed with (or before) this migration.
+ *
+ * Ordering:
+ *   (a) Skip-window fast path — live index already the exact target shape
+ *       AND no document missing `deadLetter` → nothing to do (the common
+ *       case on every boot after the first converged one, including fresh
+ *       databases whose index autoIndex built correctly from day one).
+ *   (a2) Ambiguity pre-check, ZERO writes — abort loud (see above) if any
+ *       document missing `deadLetter` carries dead-letter-era evidence.
+ *   (b) Backfill `deadLetter: false` onto field-less documents. Runs BEFORE
+ *       the index swap so the recreated partial index covers them from its
+ *       first build; safe at any point — no unique constraint is involved
+ *       anywhere in this migration, so there is no duplicate-key window and
+ *       no E11000 to convert (unlike the sibling unique-index migrations).
+ *   (c) Drop any same-key or same-name index — the legacy plain TTL, a
+ *       boot-built twin, or a divergent hand-fix under another name.
+ *   (d) Recreate `processedAt_1` with the exact schema spec. The only cost
+ *       of the (c)-(d) window is TTL expiry pausing for its duration, which
+ *       is harmless — the TTL monitor only sweeps periodically anyway.
+ * Idempotent on re-run: a second `up()` takes the fast path (a).
+ *
+ * @returns {Promise<void>}
+ */
+export async function up() {
+  const events = mongoose.connection.db.collection('processedstripeevents');
+
+  // Snapshot existing indexes once (listIndexes throws if the collection does
+  // not exist yet — tolerate that: a fresh DB has no processedstripeevents
+  // collection and autoIndex will create the index from the schema declaration).
+  let existing = [];
+  try {
+    existing = await events.listIndexes().toArray();
+  } catch (err) {
+    if (err?.codeName === 'NamespaceNotFound' || err?.code === 26) {
+      console.info('[migration] processed-stripe-event-ttl-index-partial-filter: processedstripeevents collection does not exist yet — nothing to migrate');
+      return;
+    }
+    throw err;
+  }
+
+  // ── (a) Skip-window fast path ── exact target spec already live AND no
+  // document left outside the partial filter's reach → no-op read.
+  if (existing.some(isExactTargetIndex)) {
+    const pending = await events.findOne({ deadLetter: { $exists: false } }, { projection: { _id: 1 } });
+    if (!pending) {
+      console.info('[migration] processed-stripe-event-ttl-index-partial-filter: index already matches the target spec and nothing left to backfill — skipping entirely');
+      return;
+    }
+  }
+
+  // ── (a2) Ambiguity pre-check FIRST — zero writes so far. A document
+  // missing `deadLetter` but carrying dead-letter-era evidence is impossible
+  // via any code path (the fields shipped together and the schema default
+  // materializes `deadLetter` on every write since) — if one exists anyway
+  // (manual surgery, partial restore), guessing `false` could mark a genuine
+  // permanent rejection purgeable. Abort loud; an operator classifies by hand.
+  // Note `$ne: null` deliberately does NOT match a missing `lastError` —
+  // pre-mechanism documents carry none of these fields and sail through.
+  const ambiguous = await events
+    .find(
+      { deadLetter: { $exists: false }, $or: [{ attempts: { $gt: 0 } }, { lastError: { $ne: null } }] },
+      { projection: { _id: 1 }, limit: 10 },
+    )
+    .toArray();
+  if (ambiguous.length > 0) {
+    throw new Error(
+      `[migration] processed-stripe-event-ttl-index-partial-filter ABORTED: ${ambiguous.length}+ document(s) missing 'deadLetter' but carrying dead-letter-era evidence (attempts/lastError) — ` +
+        `impossible via any code path, so this migration will NOT guess whether they are permanent rejections. ` +
+        `Classify them by hand (set deadLetter true/false explicitly) and re-run. Sample ids: ${ambiguous.map((d) => d._id).join(',')}`,
+    );
+  }
+
+  // ── (b) Backfill the discriminator onto pre-`deadLetter` documents ──
+  // Safe now: (a2) proved every remaining field-less document predates the
+  // dead-letter mechanism (a plain success claim), and a partial filter's
+  // `$eq: false` never matches a missing field — without this they would
+  // never expire.
+  const backfillResult = await events.updateMany({ deadLetter: { $exists: false } }, { $set: { deadLetter: false } });
+  if (backfillResult.modifiedCount > 0) {
+    console.info(`[migration] processed-stripe-event-ttl-index-partial-filter: backfilled deadLetter:false on ${backfillResult.modifiedCount} document(s)`);
+  }
+
+  // ── (c) Drop the legacy/divergent index(es) ── the plain TTL under the
+  // canonical name, or any same-key index living under another name.
+  for (const ix of existing) {
+    if (ix.name === '_id_') continue;
+    if (sameKey(ix) || ix.name === INDEX_NAME) {
+      await events.dropIndex(ix.name);
+      console.info(`[migration] processed-stripe-event-ttl-index-partial-filter: dropped index '${ix.name}' (legacy plain TTL or divergent spec)`);
+    }
+  }
+
+  // ── (d) Recreate the TTL index with the exact schema spec ── non-unique,
+  // so no duplicate-key failure mode exists here; the (c)-(d) window only
+  // pauses TTL expiry briefly.
+  await events.createIndex(INDEX_KEY, {
+    name: INDEX_NAME,
+    expireAfterSeconds: TTL_SECONDS,
+    partialFilterExpression: { deadLetter: { $eq: false } },
+  });
+  console.info('[migration] processed-stripe-event-ttl-index-partial-filter: created dead-letter-preserving TTL index on processedAt');
+}
+
+/**
+ * Down: no-op (warn). The pre-fix state was a TTL index that purged
+ * dead-letter documents — permanent rejections that must never be re-opened
+ * to replay. Restoring it would reintroduce the double-processing risk.
+ * Rollback = revert the schema declaration deliberately, then swap the index
+ * by hand if truly needed. The `deadLetter: false` backfill is left in place
+ * (it matches the schema default; no reader distinguishes backfilled from
+ * organically-written `false`).
+ *
+ * @returns {void}
+ */
+export function down() {
+  console.warn(
+    '[migration] processed-stripe-event-ttl-index-partial-filter DOWN: no-op; swap the processedAt TTL index manually only alongside a deliberate schema revert',
+  );
+}

modules/billing/tests/billing.processedStripeEvent.ttlIndexPartialFilter.migration.integration.tests.js

@@ -0,0 +1,201 @@
+/**
+ * Module dependencies.
+ */
+import mongoose from 'mongoose';
+
+import { jest, beforeAll, afterAll, describe, test, expect } from '@jest/globals';
+import { bootstrap } from '../../../lib/app.js';
+import mongooseService from '../../../lib/services/mongoose.js';
+
+import { up } from '../migrations/20260729120000-fix-processed-stripe-event-ttl-index-partial-filter.js';
+
+const INDEX_NAME = 'processedAt_1';
+const INDEX_KEY = { processedAt: 1 };
+const TTL_SECONDS = 30 * 24 * 60 * 60;
+const EVENT_PREFIX = 'evt_ttl_partial_migration_test_';
+
+/**
+ * Migration `20260729120000-fix-processed-stripe-event-ttl-index-partial-filter` (#4006).
+ *
+ * The schema's 30-day TTL index on `processedAt` gained
+ * `partialFilterExpression: { deadLetter: { $eq: false } }` with no
+ * accompanying migration. An already-deployed database still holding the
+ * plain `processedAt_1` TTL index hits IndexOptionsConflict (code 85) at
+ * every boot — tolerated since #4004, but the live index keeps purging
+ * dead-letter documents until THIS migration swaps it. Documents written
+ * before the `deadLetter` field existed also fall outside the partial filter
+ * (query-match semantics: `$eq: false` never matches a missing field) and
+ * would never expire — the migration backfills `deadLetter: false` onto them.
+ *
+ * `ensureLegacyState()` below reproduces the real already-deployed
+ * pre-migration state directly (bypassing this migration): the OLD plain TTL
+ * index live under the canonical name — the exact same-name/different-options
+ * shape that conflicts with the schema declaration at boot. Verifies the
+ * fixed end state via the RAW collection driver.
+ */
+describe('Migration processed-stripe-event-ttl-index-partial-filter:', () => {
+  let events;
+
+  beforeAll(async () => {
+    await bootstrap();
+    events = mongoose.connection.db.collection('processedstripeevents');
+  });
+
+  afterAll(async () => {
+    try {
+      await events.deleteMany({ eventId: { $regex: `^${EVENT_PREFIX}` } });
+    } catch (_) { /* cleanup */ }
+    try {
+      await mongooseService.disconnect();
+    } catch (e) {
+      console.log(e);
+      expect(e).toBeFalsy();
+    }
+  });
+
+  /**
+   * @desc Finds an index by name in the processedstripeevents collection.
+   * @param {string} name - the index name to look up.
+   * @returns {Promise<Object|undefined>} the index descriptor if found, undefined otherwise.
+   */
+  const findIndex = async (name) => {
+    const indexes = await events.listIndexes().toArray();
+    return indexes.find((ix) => ix.name === name);
+  };
+
+  /**
+   * @desc Simulates the real already-deployed, pre-migration state: the OLD
+   * plain TTL index (no partial filter) live under the canonical default name
+   * `processedAt_1` — the exact same-name/different-options shape the schema
+   * declaration conflicts with at boot (#4004). Used to prove `up()` swaps it
+   * to the target spec instead of leaving the wrong semantics live forever.
+   * @returns {Promise<void>}
+   */
+  const ensureLegacyState = async () => {
+    const sameKeyIndexes = (await events.listIndexes().toArray()).filter((ix) => ix.key?.processedAt === 1);
+    for (const ix of sameKeyIndexes) {
+      await events.dropIndex(ix.name);
+    }
+    await events.createIndex(INDEX_KEY, { name: INDEX_NAME, expireAfterSeconds: TTL_SECONDS });
+  };
+
+  test('up() installs the dead-letter-preserving TTL index with the exact schema spec', async () => {
+    await up();
+    const ix = await findIndex(INDEX_NAME);
+    expect(ix).toBeDefined();
+    expect(ix.key).toEqual({ processedAt: 1 });
+    expect(ix.expireAfterSeconds).toBe(TTL_SECONDS);
+    expect(ix.partialFilterExpression).toEqual({ deadLetter: { $eq: false } });
+    expect(ix.unique).toBeUndefined();
+  });
+
+  test('legacy plain TTL under the canonical name is swapped to the target spec; field-less docs get backfilled, dead-letter docs stay untouched (#4006)', async () => {
+    const preFieldId = new mongoose.Types.ObjectId();
+    const liveId = new mongoose.Types.ObjectId();
+    const deadId = new mongoose.Types.ObjectId();
+    try {
+      await ensureLegacyState();
+      // Raw driver inserts — bypass mongoose defaults so a document genuinely
+      // MISSING `deadLetter` (pre-field era) can exist.
+      await events.insertOne({ _id: preFieldId, eventId: `${EVENT_PREFIX}pre_field`, type: 'checkout.session.completed', processedAt: new Date() });
+      await events.insertOne({ _id: liveId, eventId: `${EVENT_PREFIX}live`, type: 'invoice.paid', processedAt: new Date(), deadLetter: false });
+      await events.insertOne({ _id: deadId, eventId: `${EVENT_PREFIX}dead`, type: 'invoice.paid', processedAt: new Date(), attempts: 5, deadLetter: true });
+
+      await up();
+
+      const ix = await findIndex(INDEX_NAME);
+      expect(ix).toBeDefined();
+      expect(ix.expireAfterSeconds).toBe(TTL_SECONDS);
+      expect(ix.partialFilterExpression).toEqual({ deadLetter: { $eq: false } });
+
+      // Pre-field document re-enters the TTL's scope via the backfill…
+      expect((await events.findOne({ _id: preFieldId })).deadLetter).toBe(false);
+      // …while already-written values are untouched.
+      expect((await events.findOne({ _id: liveId })).deadLetter).toBe(false);
+      expect((await events.findOne({ _id: deadId })).deadLetter).toBe(true);
+      expect((await events.findOne({ _id: deadId })).attempts).toBe(5);
+    } finally {
+      await events.deleteMany({ _id: { $in: [preFieldId, liveId, deadId] } });
+      await up(); // restore the migrated end state for the suites that follow
+    }
+  });
+
+  test('is idempotent — a second run leaves exactly one index on the key', async () => {
+    await up();
+    await up();
+    const indexes = await events.listIndexes().toArray();
+    const sameKeyIndexes = indexes.filter((ix) => ix.key?.processedAt === 1 && Object.keys(ix.key).length === 1);
+    expect(sameKeyIndexes.length).toBe(1);
+    expect(sameKeyIndexes[0].name).toBe(INDEX_NAME);
+  });
+
+  test('drops a same-key index living under another name and installs the canonical one', async () => {
+    const sameKeyIndexes = (await events.listIndexes().toArray()).filter((ix) => ix.key?.processedAt === 1);
+    for (const ix of sameKeyIndexes) {
+      await events.dropIndex(ix.name);
+    }
+    await events.createIndex(INDEX_KEY, { name: 'divergent_processedAt_index' });
+
+    await up();
+
+    expect(await findIndex('divergent_processedAt_index')).toBeUndefined();
+    const ix = await findIndex(INDEX_NAME);
+    expect(ix).toBeDefined();
+    expect(ix.partialFilterExpression).toEqual({ deadLetter: { $eq: false } });
+  });
+
+  test('ABORTS loud on a field-less document carrying dead-letter-era evidence — never guesses, backfills nothing', async () => {
+    // Impossible via any code path (deadLetter shipped with attempts/lastError
+    // and the schema default materializes it on every write since) — but
+    // manual surgery / partial restores can produce it, and flipping such a
+    // document to false could mark a genuine permanent rejection purgeable.
+    const ambiguousId = new mongoose.Types.ObjectId();
+    const plainId = new mongoose.Types.ObjectId();
+    try {
+      await ensureLegacyState();
+      await events.insertOne({ _id: ambiguousId, eventId: `${EVENT_PREFIX}ambiguous`, type: 'invoice.paid', processedAt: new Date(), attempts: 5, lastError: 'handler exploded', lastErrorAt: new Date() });
+      await events.insertOne({ _id: plainId, eventId: `${EVENT_PREFIX}plain`, type: 'invoice.paid', processedAt: new Date() });
+
+      await expect(up()).rejects.toThrow(/carrying dead-letter-era evidence/);
+
+      // Abort happened before ANY write: neither document was backfilled and
+      // the legacy index is still live (the swap never ran).
+      expect((await events.findOne({ _id: ambiguousId })).deadLetter).toBeUndefined();
+      expect((await events.findOne({ _id: plainId })).deadLetter).toBeUndefined();
+      const ix = await findIndex(INDEX_NAME);
+      expect(ix.partialFilterExpression).toBeUndefined();
+    } finally {
+      await events.deleteMany({ _id: { $in: [ambiguousId, plainId] } });
+      await up(); // restore the migrated end state for the suites that follow
+    }
+  });
+
+  test('skip-window fast path — a converged database does not touch indexes', async () => {
+    // Converge first (idempotent — also backfills any stray field-less doc,
+    // so the fast path's backfill probe finds nothing).
+    await up();
+
+    const infoSpy = jest.spyOn(console, 'info').mockImplementation(() => {});
+    try {
+      await up();
+      const messages = infoSpy.mock.calls.map((args) => args[0]);
+      expect(messages.some((m) => m.includes('skipping entirely'))).toBe(true);
+      expect(messages.some((m) => m.includes('dropped index'))).toBe(false);
+      expect(messages.some((m) => m.includes('created dead-letter-preserving'))).toBe(false);
+    } finally {
+      infoSpy.mockRestore();
+    }
+
+    const ix = await findIndex(INDEX_NAME);
+    expect(ix).toBeDefined();
+    expect(ix.partialFilterExpression).toEqual({ deadLetter: { $eq: false } });
+  });
+
+  test('schema twin is IDENTICAL — syncIndexes() has nothing to drop or rebuild', async () => {
+    await up();
+    const ProcessedStripeEvent = mongoose.model('ProcessedStripeEvent');
+    const dropped = await ProcessedStripeEvent.syncIndexes();
+    expect(dropped).toEqual([]);
+    expect(await findIndex(INDEX_NAME)).toBeDefined();
+  });
+});