diff --git a/apps/api/package.json b/apps/api/package.json index 7d5c4422a4..106cce85bf 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -98,7 +98,7 @@ "ajv-formats": "^3.0.1", "amqplib": "^0.10.9", "async-mutex": "^0.5.0", - "autumn-js": "1.0.0-beta.6", + "autumn-js": "1.0.0-beta.10", "axios": "^1.13.5", "body-parser": "^1.20.3", "bullmq": "^5.56.7", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 29868ef903..a2f7a81466 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -100,8 +100,8 @@ importers: specifier: ^0.5.0 version: 0.5.0 autumn-js: - specifier: 1.0.0-beta.6 - version: 1.0.0-beta.6(react@18.3.1) + specifier: 1.0.0-beta.10 + version: 1.0.0-beta.10(react@18.3.1) axios: specifier: ^1.13.5 version: 1.13.5 @@ -3020,8 +3020,8 @@ packages: asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - autumn-js@1.0.0-beta.6: - resolution: {integrity: sha512-OUdPvoPQVhcg/pxYuUExgiE/P2OPSvwyiTt6r7q5cTz5QEW8WSaBprc8IsEaT7ChB3usYfdh15O9cnEdgXeUvw==} + autumn-js@1.0.0-beta.10: + resolution: {integrity: sha512-Gohg4VB9npGRik0MED35kdFYHzET5NVCsVeCwsaPg11Kw9eXbbP7TDqlamaANOE+TUZ1A6RDSvgMQBrsIDrnVw==} peerDependencies: better-auth: ^1.3.17 better-call: ^1.0.12 @@ -8578,7 +8578,7 @@ snapshots: asynckit@0.4.0: {} - autumn-js@1.0.0-beta.6(react@18.3.1): + autumn-js@1.0.0-beta.10(react@18.3.1): dependencies: query-string: 9.3.1 rou3: 0.6.3 diff --git a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts index cfe251aede..e48302b044 100644 --- a/apps/api/src/services/autumn/__tests__/autumn.service.test.ts +++ b/apps/api/src/services/autumn/__tests__/autumn.service.test.ts @@ -15,6 +15,12 @@ import { jest } from "@jest/globals"; const mockTrack = jest .fn<(args: any) => Promise>() .mockResolvedValue(undefined); +const mockCheck = jest + .fn<(args: any) => Promise>() + .mockResolvedValue({ allowed: true, customerId: "org-1", balance: null }); +const mockFinalize = jest + .fn<(args: any) => Promise>() + .mockResolvedValue(undefined); const mockGetOrCreate = jest .fn<(args: any) => Promise>() .mockResolvedValue({ id: "org-1" }); @@ -24,6 +30,8 @@ const mockEntityCreate = jest.fn<(args: any) => Promise>(); const mockAutumnClient = { customers: { getOrCreate: mockGetOrCreate }, entities: { get: mockEntityGet, create: mockEntityCreate }, + balances: { finalize: mockFinalize }, + check: mockCheck, track: mockTrack, }; @@ -97,6 +105,12 @@ beforeEach(() => { jest.clearAllMocks(); autumnClientRef = mockAutumnClient; supabaseStubData = { data: { org_id: "org-1" }, error: null }; + mockCheck.mockResolvedValue({ + allowed: true, + customerId: "org-1", + balance: null, + }); + mockFinalize.mockResolvedValue(undefined); mockEntityGet.mockResolvedValue(makeEntity(0)); mockEntityCreate.mockResolvedValue({ id: "team-1" }); }); @@ -260,6 +274,71 @@ describe("ensureTrackingContext warm-cache short-circuit", () => { }); }); +// --------------------------------------------------------------------------- +// lockCredits +// --------------------------------------------------------------------------- + +describe("lockCredits", () => { + it("returns null when autumnClient is null", async () => { + autumnClientRef = null; + const svc = makeService(); + const result = await svc.lockCredits({ teamId: "team-1", value: 10 }); + expect(result).toBeNull(); + expect(mockCheck).not.toHaveBeenCalled(); + }); + + it("returns null for preview teams", async () => { + const svc = makeService(); + const result = await svc.lockCredits({ + teamId: "preview_abc", + value: 10, + }); + expect(result).toBeNull(); + expect(mockCheck).not.toHaveBeenCalled(); + }); + + it("returns the lockId on happy path", async () => { + const svc = makeService(); + + const result = await svc.lockCredits({ + teamId: "team-1", + value: 42, + lockId: "lock-123", + properties: { source: "billTeam", endpoint: "extract" }, + }); + + expect(result).toBe("lock-123"); + expect(mockCheck).toHaveBeenCalledWith( + expect.objectContaining({ + customerId: "org-1", + entityId: "team-1", + featureId: "CREDITS", + requiredBalance: 42, + properties: { source: "billTeam", endpoint: "extract" }, + lock: expect.objectContaining({ + enabled: true, + lockId: "lock-123", + }), + }), + ); + }); + + it("returns null when Autumn denies the lock", async () => { + mockCheck.mockResolvedValue({ + allowed: false, + customerId: "org-1", + balance: null, + }); + const svc = makeService(); + const result = await svc.lockCredits({ + teamId: "team-1", + value: 10, + lockId: "lock-123", + }); + expect(result).toBeNull(); + }); +}); + // --------------------------------------------------------------------------- // reserveCredits // --------------------------------------------------------------------------- @@ -303,6 +382,35 @@ describe("reserveCredits", () => { }); }); +// --------------------------------------------------------------------------- +// finalizeCreditsLock +// --------------------------------------------------------------------------- + +describe("finalizeCreditsLock", () => { + it("calls balances.finalize with confirm", async () => { + const svc = makeService(); + await svc.finalizeCreditsLock({ + lockId: "lock-123", + action: "confirm", + properties: { source: "test" }, + }); + + expect(mockFinalize).toHaveBeenCalledWith({ + lockId: "lock-123", + action: "confirm", + overrideValue: undefined, + properties: { source: "test" }, + }); + }); + + it("is a no-op when autumnClient is null", async () => { + autumnClientRef = null; + const svc = makeService(); + await svc.finalizeCreditsLock({ lockId: "lock-123", action: "release" }); + expect(mockFinalize).not.toHaveBeenCalled(); + }); +}); + // --------------------------------------------------------------------------- // refundCredits // --------------------------------------------------------------------------- @@ -402,21 +510,21 @@ describe("isAutumnEnabled", () => { }); }); -describe("experiment gate on reserveCredits", () => { +describe("experiment gate on lockCredits", () => { afterEach(() => { (config as any).AUTUMN_EXPERIMENT = "true"; (config as any).AUTUMN_EXPERIMENT_PERCENT = 100; }); - it("reserveCredits returns false when experiment is disabled", async () => { + it("lockCredits returns null when experiment is disabled", async () => { (config as any).AUTUMN_EXPERIMENT = undefined; const svc = makeService(); - const result = await svc.reserveCredits({ teamId: "team-1", value: 10 }); - expect(result).toBe(false); - expect(mockTrack).not.toHaveBeenCalled(); + const result = await svc.lockCredits({ teamId: "team-1", value: 10 }); + expect(result).toBeNull(); + expect(mockCheck).not.toHaveBeenCalled(); }); - it("reserveCredits returns false when org is outside the percent bucket", async () => { + it("lockCredits returns null when org is outside the percent bucket", async () => { // Supabase returns org whose bucket (16) is >= percent (10). supabaseStubData = { data: { org_id: "a1b2c3d4-0000-0000-0000-000000000000" }, @@ -424,12 +532,12 @@ describe("experiment gate on reserveCredits", () => { }; (config as any).AUTUMN_EXPERIMENT_PERCENT = 10; const svc = makeService(); - const result = await svc.reserveCredits({ teamId: "team-1", value: 10 }); - expect(result).toBe(false); - expect(mockTrack).not.toHaveBeenCalled(); + const result = await svc.lockCredits({ teamId: "team-1", value: 10 }); + expect(result).toBeNull(); + expect(mockCheck).not.toHaveBeenCalled(); }); - it("reserveCredits succeeds when org is inside the percent bucket", async () => { + it("lockCredits succeeds when org is inside the percent bucket", async () => { // Supabase returns org whose bucket (16) is < percent (50). supabaseStubData = { data: { org_id: "a1b2c3d4-0000-0000-0000-000000000000" }, @@ -437,9 +545,13 @@ describe("experiment gate on reserveCredits", () => { }; (config as any).AUTUMN_EXPERIMENT_PERCENT = 50; const svc = makeService(); - const result = await svc.reserveCredits({ teamId: "team-1", value: 10 }); - expect(result).toBe(true); - expect(mockTrack).toHaveBeenCalled(); + const result = await svc.lockCredits({ + teamId: "team-1", + value: 10, + lockId: "lock-123", + }); + expect(result).toBe("lock-123"); + expect(mockCheck).toHaveBeenCalled(); }); it("refundCredits still works when experiment is disabled (guard is autumnReserved)", async () => { diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.ts index ca96714472..25227fa5f2 100644 --- a/apps/api/src/services/autumn/autumn.service.ts +++ b/apps/api/src/services/autumn/autumn.service.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "crypto"; import { config } from "../../config"; import { logger } from "../../lib/logger"; import { supabase_rr_service } from "../supabase"; @@ -7,8 +8,10 @@ import type { CreateEntityResult, EnsureOrgProvisionedParams, EnsureTeamProvisionedParams, + FinalizeCreditsLockParams, GetEntityParams, GetOrCreateCustomerParams, + LockCreditsParams, TrackCreditsParams, TrackParams, } from "./types"; @@ -36,10 +39,10 @@ export function orgBucket(orgId: string): number { * stable percent gate is also evaluated so the same org always gets the * same answer. * - * Only checked at the top-level billing entry point (`reserveCredits`). - * NOT checked by `refundCredits` (already guarded by `autumnReserved`) or - * `ensureTeamProvisioned` (handled by firecrawl-web edge functions - * independently). + * Only checked at the top-level billing entry points (`lockCredits` and the + * legacy direct-track `reserveCredits`). + * NOT checked by `finalizeCreditsLock`, `refundCredits`, or + * `ensureTeamProvisioned`. */ export function isAutumnEnabled(orgId?: string): boolean { if (config.AUTUMN_EXPERIMENT !== "true") return false; @@ -324,8 +327,105 @@ export class AutumnService { } /** - * Records a credit usage event in Autumn at request time. - * Returns true on success, false if Autumn is unavailable or an error occurs. + * Reserves a team's credits in Autumn without letting Autumn gate usage. + * Returns the lock ID on success, or null if no lock was acquired. + */ + async lockCredits({ + teamId, + value, + lockId, + expiresAt, + properties, + }: LockCreditsParams): Promise { + if (!isAutumnEnabled() || !autumnClient || this.isPreviewTeam(teamId)) { + return null; + } + + const resolvedLockId = lockId ?? `billing_${randomUUID()}`; + + try { + const orgId = await this.resolveOrgId(teamId); + if (!isAutumnEnabled(orgId)) return null; + + const customerId = await this.ensureTrackingContext(teamId); + const { allowed } = await autumnClient.check({ + customerId, + entityId: teamId, + featureId: CREDITS_FEATURE_ID, + requiredBalance: value, + properties, + lock: { + enabled: true, + lockId: resolvedLockId, + expiresAt, + }, + }); + + if (!allowed) { + logger.info("Autumn lockCredits denied", { + teamId, + value, + lockId: resolvedLockId, + }); + return null; + } + + logger.info("Autumn lockCredits succeeded", { + customerId, + entityId: teamId, + featureId: CREDITS_FEATURE_ID, + value, + lockId: resolvedLockId, + properties, + }); + return resolvedLockId; + } catch (error) { + logger.warn("Autumn lockCredits failed", { + teamId, + value, + lockId: resolvedLockId, + error, + }); + return null; + } + } + + /** + * Finalizes a previously-acquired Autumn lock. + */ + async finalizeCreditsLock({ + lockId, + action, + overrideValue, + properties, + }: FinalizeCreditsLockParams): Promise { + if (!autumnClient) return; + + try { + await autumnClient.balances.finalize({ + lockId, + action, + overrideValue, + properties, + }); + logger.info("Autumn finalizeCreditsLock succeeded", { + lockId, + action, + overrideValue, + }); + } catch (error) { + logger.warn("Autumn finalizeCreditsLock failed", { + lockId, + action, + overrideValue, + error, + }); + } + } + + /** + * Records a credit usage event directly in Autumn for flows that do not + * acquire a request-time lock. Returns true on success. * * The experiment gate is evaluated here — once per request — using a stable * bucket derived from the org UUID so the same org always gets the same diff --git a/apps/api/src/services/autumn/types.ts b/apps/api/src/services/autumn/types.ts index 9b72346d58..ec05e5c5cc 100644 --- a/apps/api/src/services/autumn/types.ts +++ b/apps/api/src/services/autumn/types.ts @@ -37,6 +37,21 @@ export type EnsureTeamProvisionedParams = { name?: string | null; }; +export type LockCreditsParams = { + teamId: string; + value: number; + lockId?: string; + expiresAt?: number; + properties?: Record; +}; + +export type FinalizeCreditsLockParams = { + lockId: string; + action: "confirm" | "release"; + overrideValue?: number; + properties?: Record; +}; + export type TrackCreditsParams = { teamId: string; value: number; diff --git a/apps/api/src/services/billing/__tests__/batch_billing.test.ts b/apps/api/src/services/billing/__tests__/batch_billing.test.ts new file mode 100644 index 0000000000..0b6e617e84 --- /dev/null +++ b/apps/api/src/services/billing/__tests__/batch_billing.test.ts @@ -0,0 +1,250 @@ +import { jest } from "@jest/globals"; + +const captureException = jest.fn(); +jest.mock("@sentry/node", () => ({ + captureException, +})); + +const logger = { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + child: jest.fn(() => logger), +}; +jest.mock("../../../lib/logger", () => ({ + logger, +})); + +const withAuth = jest.fn((fn: any) => fn); +jest.mock("../../../lib/withAuth", () => ({ + withAuth, +})); + +const reserveCredits = jest.fn<(args: any) => Promise>(); +const finalizeCreditsLock = jest.fn<(args: any) => Promise>(); +jest.mock("../../autumn/autumn.service", () => ({ + autumnService: { + reserveCredits, + finalizeCreditsLock, + }, +})); + +const rpc = jest.fn<(name: string, args: any) => Promise>(); +jest.mock("../../supabase", () => ({ + supabase_service: { + rpc, + }, +})); + +const setCachedACUC = jest.fn(); +const setCachedACUCTeam = jest.fn(); +jest.mock("../../../controllers/auth", () => ({ + setCachedACUC, + setCachedACUCTeam, +})); + +let queue: string[] = []; +const billedTeams = new Set(); +const locks = new Map(); +const redis = { + set: jest.fn( + async ( + key: string, + value: string, + mode: string, + timeout: number, + nx: string, + ) => { + if ( + key !== "billing_batch_lock" || + value !== "1" || + mode !== "PX" || + timeout !== 30000 || + nx !== "NX" + ) { + throw new Error("unexpected redis.set args"); + } + if (locks.has(key)) return null; + locks.set(key, value); + return "OK"; + }, + ), + del: jest.fn(async (key: string) => { + if (key !== "billing_batch_lock") { + throw new Error("unexpected redis.del key"); + } + return locks.delete(key) ? 1 : 0; + }), + lpop: jest.fn(async (key: string) => { + if (key !== "billing_batch") { + throw new Error("unexpected redis.lpop key"); + } + return queue.shift() ?? null; + }), + llen: jest.fn(async (key: string) => { + if (key !== "billing_batch") { + throw new Error("unexpected redis.llen key"); + } + return queue.length; + }), + rpush: jest.fn(async (key: string, value: string) => { + if (key !== "billing_batch") { + throw new Error("unexpected redis.rpush key"); + } + queue.push(value); + return queue.length; + }), + sadd: jest.fn(async (key: string, teamId: string) => { + if (key !== "billed_teams") { + throw new Error("unexpected redis.sadd key"); + } + billedTeams.add(teamId); + return 1; + }), +}; +jest.mock("../../queue-service", () => ({ + getRedisConnection: () => redis, +})); + +import { processBillingBatch } from "../batch_billing"; + +function makeOp(overrides: Record = {}) { + return JSON.stringify({ + team_id: "team-1", + subscription_id: "sub-1", + credits: 10, + billing: { endpoint: "extract" }, + is_extract: false, + timestamp: "2026-03-13T00:00:00.000Z", + api_key_id: 123, + autumnLockId: null, + autumnProperties: { + source: "billTeam", + endpoint: "extract", + apiKeyId: 123, + }, + ...overrides, + }); +} + +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +beforeEach(() => { + jest.clearAllMocks(); + queue = []; + billedTeams.clear(); + locks.clear(); + reserveCredits.mockResolvedValue(true); + finalizeCreditsLock.mockResolvedValue(undefined); + rpc.mockResolvedValue({ data: [], error: null }); +}); + +describe("processBillingBatch", () => { + it("awaits lock confirmation before tracking unlocked credits", async () => { + const finalize = deferred(); + finalizeCreditsLock.mockReturnValueOnce(finalize.promise); + queue = [ + makeOp({ credits: 7, autumnLockId: "lock-1" }), + makeOp({ credits: 3, autumnLockId: null }), + ]; + + const run = processBillingBatch(); + await new Promise(resolve => setImmediate(resolve)); + + expect(finalizeCreditsLock).toHaveBeenCalledWith({ + lockId: "lock-1", + action: "confirm", + properties: expect.objectContaining({ + source: "billTeam", + apiKeyId: 123, + subscriptionId: "sub-1", + finalizeSource: "processBillingBatch", + }), + }); + expect(reserveCredits).not.toHaveBeenCalled(); + + finalize.resolve(); + await run; + + expect(reserveCredits).toHaveBeenCalledWith({ + teamId: "team-1", + value: 3, + properties: expect.objectContaining({ + source: "processBillingBatch", + apiKeyId: 123, + subscriptionId: "sub-1", + }), + }); + }); + + it("releases Autumn locks when billing returns success false", async () => { + queue = [makeOp({ autumnLockId: "lock-1" })]; + rpc.mockResolvedValueOnce({ data: null, error: new Error("db failed") }); + + await processBillingBatch(); + + expect(finalizeCreditsLock).toHaveBeenCalledWith({ + lockId: "lock-1", + action: "release", + properties: expect.objectContaining({ + source: "billTeam", + finalizeSource: "processBillingBatch_failure", + }), + }); + expect(reserveCredits).not.toHaveBeenCalled(); + }); + + it("releases Autumn locks when billing throws", async () => { + queue = [makeOp({ autumnLockId: "lock-1" })]; + rpc.mockRejectedValueOnce(new Error("rpc exploded")); + + await processBillingBatch(); + + expect(finalizeCreditsLock).toHaveBeenCalledWith({ + lockId: "lock-1", + action: "release", + properties: expect.objectContaining({ + source: "billTeam", + finalizeSource: "processBillingBatch_exception", + }), + }); + expect(captureException).toHaveBeenCalled(); + }); + + it("treats undefined autumnLockId as unlocked for legacy queued ops", async () => { + queue = [makeOp({ autumnLockId: undefined })]; + + await processBillingBatch(); + + expect(finalizeCreditsLock).not.toHaveBeenCalled(); + expect(reserveCredits).toHaveBeenCalledWith({ + teamId: "team-1", + value: 10, + properties: expect.objectContaining({ + source: "processBillingBatch", + }), + }); + }); + + it("survives unexpected synchronous finalizeAutumnLocks failures", async () => { + queue = [makeOp({ autumnLockId: "lock-1" })]; + finalizeCreditsLock.mockImplementationOnce(() => { + throw new Error("sync finalize failure"); + }); + + await processBillingBatch(); + + expect(logger.warn).toHaveBeenCalledWith( + "Autumn finalizeAutumnLocks failed unexpectedly", + expect.objectContaining({ team_id: "team-1", action: "confirm" }), + ); + }); +}); diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.ts index 04b8592087..b7f734a0f5 100644 --- a/apps/api/src/services/billing/batch_billing.ts +++ b/apps/api/src/services/billing/batch_billing.ts @@ -30,8 +30,9 @@ interface BillingOperation { is_extract: boolean; timestamp: string; api_key_id: number | null; - /** True if credits were pre-reserved in Autumn at request time via reserveCredits(). */ - autumnReserved: boolean; + /** Autumn lock ID acquired at request time, if any. */ + autumnLockId: string | null; + autumnProperties?: Record; } // Grouped billing operations for batch processing @@ -64,12 +65,62 @@ async function releaseLock() { logger.info("🔓 Released billing batch processing lock"); } +async function finalizeAutumnLocks( + group: GroupedBillingOperation, + action: "confirm" | "release", + source: string, +) { + const lockedOperations = group.operations.filter( + ( + op, + ): op is BillingOperation & { + autumnLockId: string; + } => typeof op.autumnLockId === "string", + ); + + if (lockedOperations.length === 0) return; + + try { + const results = await Promise.allSettled( + lockedOperations.map(op => + autumnService.finalizeCreditsLock({ + lockId: op.autumnLockId, + action, + properties: { + ...op.autumnProperties, + subscriptionId: op.subscription_id, + finalizeSource: source, + }, + }), + ), + ); + + const rejectedCount = results.filter( + result => result.status === "rejected", + ).length; + if (rejectedCount > 0) { + logger.warn("Autumn finalizeCreditsLock rejected unexpectedly", { + team_id: group.team_id, + action, + rejectedCount, + }); + } + } catch (error) { + logger.warn("Autumn finalizeAutumnLocks failed unexpectedly", { + team_id: group.team_id, + action, + operation_count: lockedOperations.length, + error, + }); + } +} + /** * Dequeues pending billing operations from Redis, groups them by team, and * commits each group to Supabase via the `bill_team_6` RPC. * - * For groups where credits were pre-reserved in Autumn (`autumnReserved: true`), - * a refund is issued on failure; nothing is done on success. For unreserved + * For groups where credits were locked in Autumn (`autumnLockId != null`), + * the lock is confirmed on success and released on failure. For unlocked * groups (legacy / BullMQ path), Autumn is updated post-commit. */ export async function processBillingBatch() { @@ -127,7 +178,7 @@ export async function processBillingBatch() { } // Process each group of operations - for (const [key, group] of groupedOperations.entries()) { + for (const [, group] of groupedOperations.entries()) { logger.info( `🔄 Billing team ${group.team_id} for ${group.total_credits} credits`, { @@ -146,12 +197,9 @@ export async function processBillingBatch() { continue; } - // Compute per-group credit split before billing so the catch block can - // issue a refund even if supaBillTeam throws. - const reservedCredits = group.operations - .filter(op => op.autumnReserved) + const unreservedCredits = group.operations + .filter(op => op.autumnLockId == null) .reduce((sum, op) => sum + op.credits, 0); - const unreservedCredits = group.total_credits - reservedCredits; try { // Execute the actual billing @@ -176,19 +224,11 @@ export async function processBillingBatch() { credits: group.total_credits, }, ); - // Refund only the credits that were actually reserved in Autumn. - if (reservedCredits > 0) { - void autumnService.refundCredits({ - teamId: group.team_id, - value: reservedCredits, - properties: { - source: "processBillingBatch_failure", - ...toAutumnBillingProperties(group.billing), - apiKeyId: group.api_key_id, - subscriptionId: group.subscription_id, - }, - }); - } + await finalizeAutumnLocks( + group, + "release", + "processBillingBatch_failure", + ); continue; } @@ -196,10 +236,12 @@ export async function processBillingBatch() { `✅ Successfully billed team ${group.team_id} for ${group.total_credits} credits`, ); - // Track only unreserved credits post-commit; reserved credits were - // already recorded in Autumn at request time. + await finalizeAutumnLocks(group, "confirm", "processBillingBatch"); + + // Track only unlocked credits post-commit; locked credits are + // confirmed above and should not be re-tracked. if (unreservedCredits > 0) { - void autumnService.reserveCredits({ + await autumnService.reserveCredits({ teamId: group.team_id, value: unreservedCredits, properties: { @@ -222,19 +264,11 @@ export async function processBillingBatch() { credits: group.total_credits, }, }); - // Billing threw before committing — refund any Autumn-reserved credits. - if (reservedCredits > 0) { - void autumnService.refundCredits({ - teamId: group.team_id, - value: reservedCredits, - properties: { - source: "processBillingBatch_exception", - ...toAutumnBillingProperties(group.billing), - apiKeyId: group.api_key_id, - subscriptionId: group.subscription_id, - }, - }); - } + await finalizeAutumnLocks( + group, + "release", + "processBillingBatch_exception", + ); } } @@ -271,9 +305,9 @@ export function startBillingBatchProcessing() { /** * Enqueues a billing operation for async batch processing. * - * Pass `autumnReserved: true` if credits were already reserved in Autumn via - * `autumnService.reserveCredits()` — the batch processor will refund on - * `bill_team_6` failure and skip re-tracking on success. + * Pass `autumnLockId` when credits were locked in Autumn via + * `autumnService.lockCredits()` — the batch processor will release on + * `bill_team_6` failure and confirm on success. */ export async function queueBillingOperation( team_id: string, @@ -282,7 +316,8 @@ export async function queueBillingOperation( api_key_id: number | null, billing: BillingMetadata, is_extract: boolean = false, - autumnReserved: boolean = false, + autumnLockId: string | null = null, + autumnProperties?: Record, ) { // Skip queuing for preview teams if (team_id === "preview" || team_id.startsWith("preview_")) { @@ -307,7 +342,8 @@ export async function queueBillingOperation( is_extract, timestamp: new Date().toISOString(), api_key_id, - autumnReserved, + autumnLockId, + autumnProperties, }; // Add operation to Redis list diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.ts index 7bcc547e8e..52d63bea43 100644 --- a/apps/api/src/services/billing/credit_billing.ts +++ b/apps/api/src/services/billing/credit_billing.ts @@ -9,10 +9,7 @@ import { autoCharge } from "./auto_charge"; import { getValue, setValue } from "../redis"; import { queueBillingOperation } from "./batch_billing"; import { autumnService } from "../autumn/autumn.service"; -import { - toAutumnBillingProperties, - type BillingMetadata, -} from "./types"; +import { toAutumnBillingProperties, type BillingMetadata } from "./types"; import type { Logger } from "winston"; /** @@ -35,16 +32,17 @@ export async function billTeam( billing: BillingMetadata, logger: Logger | undefined, ) => { - // Reserve in Autumn at request time; await so autumnReserved is accurate. - // billTeam is fire-and-forget at call sites, so this doesn't block responses. - const autumnReserved = await autumnService.reserveCredits({ + const autumnProperties = { + source: "billTeam", + ...toAutumnBillingProperties(billing), + apiKeyId: api_key_id, + }; + // Acquire an Autumn lock opportunistically, but never gate usage on it. + // billTeam is fire-and-forget at call sites, so this does not block responses. + const autumnLockId = await autumnService.lockCredits({ teamId: team_id, value: credits, - properties: { - source: "billTeam", - ...toAutumnBillingProperties(billing), - apiKeyId: api_key_id, - }, + properties: autumnProperties, }); return queueBillingOperation( team_id, @@ -53,7 +51,8 @@ export async function billTeam( api_key_id, billing, false, - autumnReserved, + autumnLockId, + autumnProperties, ); }, { success: true, message: "No DB, bypassed." }, diff --git a/apps/ui/ingestion-ui/package.json b/apps/ui/ingestion-ui/package.json index 950f9ba7ab..43e3632773 100644 --- a/apps/ui/ingestion-ui/package.json +++ b/apps/ui/ingestion-ui/package.json @@ -44,6 +44,7 @@ "pnpm": { "overrides": { "ajv": "^8.18.0", + "flatted@<3.3.4": ">=3.3.4", "minimatch@<10.2.3": ">=10.2.3", "rollup@<4.59.0": ">=4.59.0" } diff --git a/apps/ui/ingestion-ui/pnpm-lock.yaml b/apps/ui/ingestion-ui/pnpm-lock.yaml index 26bd706090..4893389a54 100644 --- a/apps/ui/ingestion-ui/pnpm-lock.yaml +++ b/apps/ui/ingestion-ui/pnpm-lock.yaml @@ -6,6 +6,7 @@ settings: overrides: ajv: ^8.18.0 + flatted@<3.3.4: '>=3.3.4' minimatch@<10.2.3: '>=10.2.3' rollup@<4.59.0: '>=4.59.0' @@ -1218,8 +1219,8 @@ packages: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} - flatted@3.3.3: - resolution: {integrity: sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg==} + flatted@3.4.1: + resolution: {integrity: sha512-IxfVbRFVlV8V/yRaGzk0UVIcsKKHMSfYw66T/u4nTwlWteQePsxe//LjudR1AMX4tZW3WFCh3Zqa/sjlqpbURQ==} fraction.js@5.3.4: resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} @@ -2735,10 +2736,10 @@ snapshots: flat-cache@4.0.1: dependencies: - flatted: 3.3.3 + flatted: 3.4.1 keyv: 4.5.4 - flatted@3.3.3: {} + flatted@3.4.1: {} fraction.js@5.3.4: {} diff --git a/examples/kubernetes/firecrawl-helm/Chart.yaml b/examples/kubernetes/firecrawl-helm/Chart.yaml index dc97ecad86..98ec33428e 100644 --- a/examples/kubernetes/firecrawl-helm/Chart.yaml +++ b/examples/kubernetes/firecrawl-helm/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v2 name: firecrawl description: A Helm chart for deploying the Firecrawl application type: application -version: 0.1.0 +version: 0.2.0 diff --git a/examples/kubernetes/firecrawl-helm/README.md b/examples/kubernetes/firecrawl-helm/README.md index f8744818d7..5cdf1f34b0 100644 --- a/examples/kubernetes/firecrawl-helm/README.md +++ b/examples/kubernetes/firecrawl-helm/README.md @@ -1,51 +1,120 @@ # Firecrawl Helm Chart -This Helm chart deploys Firecrawl components (API, Worker, Playwright service, Redis, etc.) on a Kubernetes cluster using environment-specific overlays. -## Pre-deployment +This chart deploys Firecrawl on Kubernetes with: +- `api` +- `worker` (queue-worker) +- `extract-worker` +- `nuq-worker` +- `nuq-prefetch-worker` +- `playwright-service` +- `redis` +- `nuq-postgres` +- `rabbitmq` -1. **Configure Secrets and ConfigMaps:** - - Update `values.yaml` with the necessary environment-specific values in the `overlays//values.yaml` file. - - **Note:** If using `REDIS_PASSWORD`, adjust the `REDIS_URL` and `REDIS_RATE_LIMIT_URL` in the ConfigMap to include the password. +## Image Strategy -2. **Build and Push Docker Images:** - - API/Worker: - ```bash - docker build --no-cache --platform linux/amd64 -t ghcr.io/winkk-dev/firecrawl:latest ../../../apps/api - docker push ghcr.io/winkk-dev/firecrawl:latest - ``` - - Playwright Service: - ```bash - docker build --no-cache --platform linux/amd64 -t ghcr.io/winkk-dev/firecrawl-playwright:latest ../../../apps/playwright-service - docker push ghcr.io/winkk-dev/firecrawl-playwright:latest - ``` +- **x86-only cluster**: use official Firecrawl images from GHCR (`ghcr.io/firecrawl/...`). +- **ARM or mixed ARM+x86 cluster**: use your multi-arch `winkkgmbh` images. -## Deployment +Official Firecrawl images are fine for x86. Use winkk images only when ARM support is needed. + +## Configure Values + +Use `values.yaml` plus one overlay. + +Important fields: +- `secret.*` for API keys and sensitive values. +- `config.extra` / `secret.extra` for custom env vars. +- `image.dockerSecretEnabled` and `imagePullSecrets` for private registries. +- `resources.enabled` enables/disables all container resource requests/limits. + Default: `false`. +- `rabbitmq.enabled`, `extractWorker.enabled`, `nuqPrefetchWorker.enabled` to toggle components. + +## Deploy + +Render: -Render the manifests for review: ```bash -helm template winkk-ai . -f values.yaml -f overlays/dev/values.yaml -n winkk-ai +HELM_NO_PLUGINS=1 helm template firecrawl . \ + -f values.yaml \ + -f overlays/prod/values.yaml \ + -n firecrawl ``` -Deploy or upgrade the release: +Install/upgrade: + ```bash -helm upgrade firecrawl . -f values.yaml -f overlays/dev/values.yaml -n firecrawl --install --create-namespace +HELM_NO_PLUGINS=1 helm upgrade firecrawl . \ + -f values.yaml \ + -f overlays/prod/values.yaml \ + -n firecrawl \ + --install \ + --create-namespace ``` -## Testing +### Use Official Firecrawl Images (x86-only) + +If your cluster is x86-only and you want official images, override repositories: -Forward the API service port to your local machine: ```bash -kubectl port-forward svc/firecrawl-api 3002:3002 -n firecrawl +HELM_NO_PLUGINS=1 helm upgrade firecrawl . \ + -f values.yaml \ + -f overlays/prod/values.yaml \ + --set image.repository=ghcr.io/firecrawl/firecrawl \ + --set playwright.repository=ghcr.io/firecrawl/playwright-service \ + --set nuqPostgres.image.repository=ghcr.io/firecrawl/nuq-postgres \ + -n firecrawl \ + --install \ + --create-namespace ``` -## Cleanup +## Build and Push Multi-Arch Containers (ARM+x86) + +Run from `examples/kubernetes/firecrawl-helm`: -To uninstall the deployment: ```bash -helm uninstall firecrawl -n firecrawl +docker buildx create --name multiarch --use --bootstrap +``` + +```bash +docker buildx build --platform linux/amd64,linux/arm64 --push \ + -t docker.io/winkkgmbh/firecrawl:latest \ + ../../../apps/api + +docker buildx build --platform linux/amd64,linux/arm64 --push \ + -t docker.io/winkkgmbh/firecrawl-playwright:latest \ + ../../../apps/playwright-service-ts + +docker buildx build --platform linux/amd64,linux/arm64 --push \ + -t docker.io/winkkgmbh/nuq-postgres:latest \ + ../../../apps/nuq-postgres ``` + +## Package and Push Helm Chart (OCI) + +```bash +HELM_NO_PLUGINS=1 helm package . --destination /tmp/helm-packages +HELM_NO_PLUGINS=1 helm push /tmp/helm-packages/firecrawl-0.2.0.tgz oci://registry-1.docker.io/winkkgmbh +``` + +Install from OCI: + +```bash +HELM_NO_PLUGINS=1 helm upgrade --install firecrawl oci://registry-1.docker.io/winkkgmbh/firecrawl \ + --version 0.2.0 \ + -n firecrawl --create-namespace \ + -f values.yaml \ + -f overlays/prod/values.yaml ``` ---- +## Test -This README provides a quick guide to configuring, deploying, testing, and cleaning up your Firecrawl installation using Helm. \ No newline at end of file +```bash +kubectl port-forward svc/firecrawl-firecrawl-api 3002:3002 -n firecrawl +``` + +## Cleanup + +```bash +helm uninstall firecrawl -n firecrawl +``` diff --git a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml b/examples/kubernetes/firecrawl-helm/templates/configmap.yaml index 7d69a86910..e36e1fc610 100644 --- a/examples/kubernetes/firecrawl-helm/templates/configmap.yaml +++ b/examples/kubernetes/firecrawl-helm/templates/configmap.yaml @@ -1,14 +1,50 @@ +{{- $fullname := include "firecrawl.fullname" . -}} +{{- $redisDefault := printf "redis://%s-redis:%v" $fullname .Values.service.redis.port -}} +{{- $playwrightDefault := printf "http://%s-playwright:%v/scrape" $fullname .Values.service.playwright.port -}} +{{- $pgUser := default "postgres" .Values.nuqPostgres.auth.username -}} +{{- $pgPassword := default "password" .Values.nuqPostgres.auth.password -}} +{{- $pgDb := default "postgres" .Values.nuqPostgres.auth.database -}} +{{- $nuqDbDefault := printf "postgresql://%s:%s@%s-nuq-postgres:5432/%s" $pgUser $pgPassword $fullname $pgDb -}} +{{- $nuqDbUrl := default $nuqDbDefault .Values.config.NUQ_DATABASE_URL -}} +{{- $rabbitMqDefault := printf "amqp://%s-rabbitmq:%v" $fullname .Values.service.rabbitmq.port -}} +{{- $apiHostDefault := printf "%s-api" $fullname -}} +{{- $apiPortDefault := printf "%v" .Values.service.api.port -}} apiVersion: v1 kind: ConfigMap metadata: name: {{ include "firecrawl.fullname" . }}-config data: NUM_WORKERS_PER_QUEUE: {{ .Values.config.NUM_WORKERS_PER_QUEUE | quote }} - PORT: {{ .Values.config.PORT | quote }} + PORT: {{ default (printf "%v" .Values.service.api.port) .Values.config.PORT | quote }} HOST: {{ .Values.config.HOST | quote }} - REDIS_URL: {{ .Values.config.REDIS_URL | quote }} - REDIS_RATE_LIMIT_URL: {{ .Values.config.REDIS_RATE_LIMIT_URL | quote }} - PLAYWRIGHT_MICROSERVICE_URL: {{ .Values.config.PLAYWRIGHT_MICROSERVICE_URL | quote }} + WORKER_PORT: {{ default (printf "%v" .Values.worker.port) .Values.config.WORKER_PORT | quote }} + EXTRACT_WORKER_PORT: {{ default (printf "%v" .Values.extractWorker.port) .Values.config.EXTRACT_WORKER_PORT | quote }} + NUQ_WORKER_PORT: {{ default (printf "%v" .Values.nuqWorker.port) .Values.config.NUQ_WORKER_PORT | quote }} + NUQ_PREFETCH_WORKER_PORT: {{ default (printf "%v" .Values.nuqPrefetchWorker.port) .Values.config.NUQ_PREFETCH_WORKER_PORT | quote }} + NUQ_WORKER_COUNT: {{ default (printf "%v" .Values.nuqWorker.replicaCount) .Values.config.NUQ_WORKER_COUNT | quote }} + REDIS_URL: {{ default $redisDefault .Values.config.REDIS_URL | quote }} + REDIS_RATE_LIMIT_URL: {{ default $redisDefault .Values.config.REDIS_RATE_LIMIT_URL | quote }} + PLAYWRIGHT_MICROSERVICE_URL: {{ default $playwrightDefault .Values.config.PLAYWRIGHT_MICROSERVICE_URL | quote }} + NUQ_DATABASE_URL: {{ $nuqDbUrl | quote }} + NUQ_DATABASE_URL_LISTEN: {{ default $nuqDbUrl .Values.config.NUQ_DATABASE_URL_LISTEN | quote }} + NUQ_RABBITMQ_URL: {{ default $rabbitMqDefault .Values.config.NUQ_RABBITMQ_URL | quote }} USE_DB_AUTHENTICATION: {{ .Values.config.USE_DB_AUTHENTICATION | quote }} - HDX_NODE_BETA_MODE: {{ .Values.config.HDX_NODE_BETA_MODE | quote }} - NUQ_DATABASE_URL: {{ .Values.config.NUQ_DATABASE_URL | quote }} + IS_KUBERNETES: {{ .Values.config.IS_KUBERNETES | quote }} + ENV: {{ .Values.config.ENV | quote }} + LOGGING_LEVEL: {{ .Values.config.LOGGING_LEVEL | quote }} + FIRECRAWL_APP_SCHEME: {{ .Values.config.FIRECRAWL_APP_SCHEME | quote }} + FIRECRAWL_APP_HOST: {{ default $apiHostDefault .Values.config.FIRECRAWL_APP_HOST | quote }} + FIRECRAWL_APP_PORT: {{ default $apiPortDefault .Values.config.FIRECRAWL_APP_PORT | quote }} + OPENAI_BASE_URL: {{ .Values.config.OPENAI_BASE_URL | quote }} + MODEL_NAME: {{ .Values.config.MODEL_NAME | quote }} + MODEL_EMBEDDING_NAME: {{ .Values.config.MODEL_EMBEDDING_NAME | quote }} + OLLAMA_BASE_URL: {{ .Values.config.OLLAMA_BASE_URL | quote }} + PROXY_SERVER: {{ .Values.config.PROXY_SERVER | quote }} + PROXY_USERNAME: {{ .Values.config.PROXY_USERNAME | quote }} + SEARXNG_ENDPOINT: {{ .Values.config.SEARXNG_ENDPOINT | quote }} + SEARXNG_ENGINES: {{ .Values.config.SEARXNG_ENGINES | quote }} + SEARXNG_CATEGORIES: {{ .Values.config.SEARXNG_CATEGORIES | quote }} + SELF_HOSTED_WEBHOOK_URL: {{ .Values.config.SELF_HOSTED_WEBHOOK_URL | quote }} + {{- range $k, $v := .Values.config.extra }} + {{ $k }}: {{ $v | quote }} + {{- end }} diff --git a/examples/kubernetes/firecrawl-helm/templates/deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/deployment.yaml index ba0be85fed..602fdf0573 100644 --- a/examples/kubernetes/firecrawl-helm/templates/deployment.yaml +++ b/examples/kubernetes/firecrawl-helm/templates/deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: {{ include "firecrawl.name" . }}-api spec: - replicas: {{ .Values.replicaCount }} + replicas: {{ .Values.api.replicaCount | default .Values.replicaCount }} selector: matchLabels: app: {{ include "firecrawl.name" . }}-api @@ -18,21 +18,31 @@ spec: imagePullSecrets: {{- toYaml .Values.imagePullSecrets | nindent 8 }} {{- end }} + terminationGracePeriodSeconds: 180 containers: - name: api image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - args: [ "pnpm", "run", "start" ] + command: [ "node" ] + args: [ "--max-old-space-size=6144", "dist/src/index.js" ] ports: - containerPort: {{ .Values.service.api.port }} env: - name: FLY_PROCESS_GROUP value: "app" + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name envFrom: - configMapRef: name: {{ include "firecrawl.fullname" . }}-config - secretRef: name: {{ include "firecrawl.fullname" . }}-secret + {{- if and .Values.resources.enabled .Values.api.resources }} + resources: + {{- toYaml .Values.api.resources | nindent 12 }} + {{- end }} livenessProbe: httpGet: path: /v0/health/liveness diff --git a/examples/kubernetes/firecrawl-helm/templates/extract-worker-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/extract-worker-deployment.yaml new file mode 100644 index 0000000000..8457a0a0d1 --- /dev/null +++ b/examples/kubernetes/firecrawl-helm/templates/extract-worker-deployment.yaml @@ -0,0 +1,67 @@ +{{- if .Values.extractWorker.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-extract-worker + labels: + app: {{ include "firecrawl.name" . }}-extract-worker +spec: + replicas: {{ .Values.extractWorker.replicaCount | default 1 }} + selector: + matchLabels: + app: {{ include "firecrawl.name" . }}-extract-worker + template: + metadata: + labels: + app: {{ include "firecrawl.name" . }}-extract-worker + spec: + {{- if .Values.image.dockerSecretEnabled }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: 60 + containers: + - name: extract-worker + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: [ "node" ] + args: [ "--max-old-space-size=3072", "dist/src/services/extract-worker.js" ] + ports: + - containerPort: {{ .Values.extractWorker.port | int }} + env: + - name: FLY_PROCESS_GROUP + value: "extract-worker" + - name: EXTRACT_WORKER_PORT + value: {{ .Values.extractWorker.port | quote }} + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + envFrom: + - configMapRef: + name: {{ include "firecrawl.fullname" . }}-config + - secretRef: + name: {{ include "firecrawl.fullname" . }}-secret + {{- if and .Values.resources.enabled .Values.extractWorker.resources }} + resources: + {{- toYaml .Values.extractWorker.resources | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /liveness + port: {{ .Values.extractWorker.port | int }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: {{ .Values.extractWorker.port | int }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 +{{- end }} diff --git a/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml index 4a5a01dcc4..df41f8ca8e 100644 --- a/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml +++ b/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-deployment.yaml @@ -26,7 +26,7 @@ spec: - name: POSTGRES_USER value: "{{ .Values.nuqPostgres.auth.username | default "postgres" }}" - name: POSTGRES_PASSWORD - value: "password" + value: "{{ .Values.nuqPostgres.auth.password | default "password" }}" - name: POSTGRES_DB value: "{{ .Values.nuqPostgres.auth.database | default "postgres" }}" ports: @@ -34,17 +34,19 @@ spec: volumeMounts: - name: postgres-storage mountPath: /var/lib/postgresql/data - {{- if .Values.nuqPostgres.resources }} + {{- if .Values.resources.enabled }} + {{- if .Values.nuqPostgres.resources }} resources: - {{- toYaml .Values.nuqPostgres.resources | nindent 12 }} - {{- else }} + {{- toYaml .Values.nuqPostgres.resources | nindent 12 }} + {{- else }} resources: - requests: - memory: "512Mi" - cpu: "250m" - limits: - memory: "1Gi" - cpu: "500m" + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" + {{- end }} {{- end }} volumes: - name: postgres-storage diff --git a/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-pvc.yaml b/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-pvc.yaml new file mode 100644 index 0000000000..59c49341d9 --- /dev/null +++ b/examples/kubernetes/firecrawl-helm/templates/nuq-postgres-pvc.yaml @@ -0,0 +1,17 @@ +{{- if .Values.nuqPostgres.persistence.enabled }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "firecrawl.fullname" . }}-nuq-postgres-pvc + labels: + app: {{ include "firecrawl.name" . }}-nuq-postgres +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.nuqPostgres.persistence.size | quote }} + {{- if .Values.nuqPostgres.persistence.storageClass }} + storageClassName: {{ .Values.nuqPostgres.persistence.storageClass | quote }} + {{- end }} +{{- end }} diff --git a/examples/kubernetes/firecrawl-helm/templates/nuq-prefetch-worker-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/nuq-prefetch-worker-deployment.yaml new file mode 100644 index 0000000000..2247e346ae --- /dev/null +++ b/examples/kubernetes/firecrawl-helm/templates/nuq-prefetch-worker-deployment.yaml @@ -0,0 +1,69 @@ +{{- if .Values.nuqPrefetchWorker.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-nuq-prefetch-worker + labels: + app: {{ include "firecrawl.name" . }}-nuq-prefetch-worker +spec: + replicas: {{ .Values.nuqPrefetchWorker.replicaCount | default 1 }} + selector: + matchLabels: + app: {{ include "firecrawl.name" . }}-nuq-prefetch-worker + template: + metadata: + labels: + app: {{ include "firecrawl.name" . }}-nuq-prefetch-worker + spec: + {{- if .Values.image.dockerSecretEnabled }} + imagePullSecrets: + {{- toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + terminationGracePeriodSeconds: 60 + containers: + - name: nuq-prefetch-worker + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: [ "node" ] + args: [ "--max-old-space-size=2048", "dist/src/services/worker/nuq-prefetch-worker.js" ] + ports: + - containerPort: {{ .Values.nuqPrefetchWorker.port | int }} + env: + - name: FLY_PROCESS_GROUP + value: "nuq-prefetch-worker" + - name: NUQ_PREFETCH_WORKER_PORT + value: {{ .Values.nuqPrefetchWorker.port | quote }} + - name: NUQ_PREFETCH_REPLICAS + value: {{ .Values.nuqPrefetchWorker.replicaCount | quote }} + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + envFrom: + - configMapRef: + name: {{ include "firecrawl.fullname" . }}-config + - secretRef: + name: {{ include "firecrawl.fullname" . }}-secret + {{- if and .Values.resources.enabled .Values.nuqPrefetchWorker.resources }} + resources: + {{- toYaml .Values.nuqPrefetchWorker.resources | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /health + port: {{ .Values.nuqPrefetchWorker.port | int }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: {{ .Values.nuqPrefetchWorker.port | int }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 +{{- end }} diff --git a/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml index d36d83403d..d019677142 100644 --- a/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml +++ b/examples/kubernetes/firecrawl-helm/templates/nuq-worker-deployment.yaml @@ -25,32 +25,40 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} command: [ "node" ] args: [ "--max-old-space-size=3072", "dist/src/services/worker/nuq-worker.js" ] + ports: + - containerPort: {{ .Values.nuqWorker.port | int }} env: - name: FLY_PROCESS_GROUP value: "nuq-worker" - name: NUQ_WORKER_PORT - value: "3006" + value: {{ .Values.nuqWorker.port | quote }} + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name envFrom: - configMapRef: name: {{ include "firecrawl.fullname" . }}-config - secretRef: name: {{ include "firecrawl.fullname" . }}-secret - {{- if .Values.nuqWorker.resources }} + {{- if .Values.resources.enabled }} + {{- if .Values.nuqWorker.resources }} resources: - {{- toYaml .Values.nuqWorker.resources | nindent 12 }} - {{- else }} + {{- toYaml .Values.nuqWorker.resources | nindent 12 }} + {{- else }} resources: - requests: - memory: "3G" - cpu: "1000m" - limits: - memory: "4G" - cpu: "1000m" + requests: + memory: "3G" + cpu: "1000m" + limits: + memory: "4G" + cpu: "1000m" + {{- end }} {{- end }} livenessProbe: httpGet: path: /health - port: 3006 + port: {{ .Values.nuqWorker.port | int }} initialDelaySeconds: 5 periodSeconds: 5 timeoutSeconds: 5 @@ -59,7 +67,7 @@ spec: readinessProbe: httpGet: path: /health - port: 3006 + port: {{ .Values.nuqWorker.port | int }} initialDelaySeconds: 5 periodSeconds: 5 timeoutSeconds: 5 diff --git a/examples/kubernetes/firecrawl-helm/templates/playwright-configmap.yaml b/examples/kubernetes/firecrawl-helm/templates/playwright-configmap.yaml index bff1866c9e..f3be190857 100644 --- a/examples/kubernetes/firecrawl-helm/templates/playwright-configmap.yaml +++ b/examples/kubernetes/firecrawl-helm/templates/playwright-configmap.yaml @@ -3,4 +3,6 @@ kind: ConfigMap metadata: name: {{ include "firecrawl.fullname" . }}-playwright-config data: - PORT: {{ .Values.playwrightConfig.PORT | quote }} + PORT: {{ default (printf "%v" .Values.service.playwright.port) .Values.playwrightConfig.PORT | quote }} + BLOCK_MEDIA: {{ .Values.playwrightConfig.BLOCK_MEDIA | quote }} + MAX_CONCURRENT_PAGES: {{ .Values.playwrightConfig.MAX_CONCURRENT_PAGES | quote }} diff --git a/examples/kubernetes/firecrawl-helm/templates/playwright-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/playwright-deployment.yaml index 8f7c1243ab..bfb1d8335e 100644 --- a/examples/kubernetes/firecrawl-helm/templates/playwright-deployment.yaml +++ b/examples/kubernetes/firecrawl-helm/templates/playwright-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: {{ include "firecrawl.name" . }}-playwright spec: - replicas: {{ .Values.replicaCount }} + replicas: {{ .Values.playwright.replicaCount | default .Values.replicaCount }} selector: matchLabels: app: {{ include "firecrawl.name" . }}-playwright @@ -25,11 +25,15 @@ spec: ports: - containerPort: {{ .Values.service.playwright.port }} envFrom: + - configMapRef: + name: {{ include "firecrawl.fullname" . }}-config + - secretRef: + name: {{ include "firecrawl.fullname" . }}-secret - configMapRef: name: {{ include "firecrawl.fullname" . }}-playwright-config livenessProbe: httpGet: - path: /health/liveness + path: /health port: {{ .Values.service.playwright.port }} initialDelaySeconds: 30 periodSeconds: 30 @@ -38,7 +42,7 @@ spec: failureThreshold: 3 readinessProbe: httpGet: - path: /health/readiness + path: /health port: {{ .Values.service.playwright.port }} initialDelaySeconds: 30 periodSeconds: 30 diff --git a/examples/kubernetes/firecrawl-helm/templates/rabbitmq-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/rabbitmq-deployment.yaml new file mode 100644 index 0000000000..ddd1d9b650 --- /dev/null +++ b/examples/kubernetes/firecrawl-helm/templates/rabbitmq-deployment.yaml @@ -0,0 +1,60 @@ +{{- if .Values.rabbitmq.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "firecrawl.fullname" . }}-rabbitmq + labels: + app: {{ include "firecrawl.name" . }}-rabbitmq +spec: + replicas: {{ .Values.rabbitmq.replicaCount | default 1 }} + selector: + matchLabels: + app: {{ include "firecrawl.name" . }}-rabbitmq + template: + metadata: + labels: + app: {{ include "firecrawl.name" . }}-rabbitmq + spec: + containers: + - name: rabbitmq + image: {{ .Values.rabbitmq.image }} + ports: + - containerPort: {{ .Values.service.rabbitmq.port }} + - containerPort: {{ .Values.service.rabbitmq.managementPort }} + {{- if and .Values.resources.enabled .Values.rabbitmq.resources }} + resources: + {{- toYaml .Values.rabbitmq.resources | nindent 12 }} + {{- end }} + livenessProbe: + tcpSocket: + port: {{ .Values.service.rabbitmq.port }} + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + readinessProbe: + tcpSocket: + port: {{ .Values.service.rabbitmq.port }} + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 6 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "firecrawl.fullname" . }}-rabbitmq +spec: + type: {{ .Values.service.rabbitmq.type }} + selector: + app: {{ include "firecrawl.name" . }}-rabbitmq + ports: + - name: amqp + protocol: TCP + port: {{ .Values.service.rabbitmq.port }} + targetPort: {{ .Values.service.rabbitmq.port }} + - name: management + protocol: TCP + port: {{ .Values.service.rabbitmq.managementPort }} + targetPort: {{ .Values.service.rabbitmq.managementPort }} +{{- end }} diff --git a/examples/kubernetes/firecrawl-helm/templates/secret.yaml b/examples/kubernetes/firecrawl-helm/templates/secret.yaml index 3840b7c4ea..585e3f5128 100644 --- a/examples/kubernetes/firecrawl-helm/templates/secret.yaml +++ b/examples/kubernetes/firecrawl-helm/templates/secret.yaml @@ -9,8 +9,13 @@ data: LLAMAPARSE_API_KEY: {{ .Values.secret.LLAMAPARSE_API_KEY | b64enc | quote }} BULL_AUTH_KEY: {{ .Values.secret.BULL_AUTH_KEY | b64enc | quote }} TEST_API_KEY: {{ .Values.secret.TEST_API_KEY | b64enc | quote }} - SCRAPING_BEE_API_KEY: {{ .Values.secret.SCRAPING_BEE_API_KEY | b64enc | quote }} - STRIPE_PRICE_ID_STANDARD: {{ .Values.secret.STRIPE_PRICE_ID_STANDARD | b64enc | quote }} - STRIPE_PRICE_ID_SCALE: {{ .Values.secret.STRIPE_PRICE_ID_SCALE | b64enc | quote }} FIRE_ENGINE_BETA_URL: {{ .Values.secret.FIRE_ENGINE_BETA_URL | b64enc | quote }} + SUPABASE_ANON_TOKEN: {{ .Values.secret.SUPABASE_ANON_TOKEN | b64enc | quote }} + SUPABASE_URL: {{ .Values.secret.SUPABASE_URL | b64enc | quote }} + SUPABASE_SERVICE_TOKEN: {{ .Values.secret.SUPABASE_SERVICE_TOKEN | b64enc | quote }} + PROXY_PASSWORD: {{ .Values.secret.PROXY_PASSWORD | b64enc | quote }} + SELF_HOSTED_WEBHOOK_HMAC_SECRET: {{ .Values.secret.SELF_HOSTED_WEBHOOK_HMAC_SECRET | b64enc | quote }} REDIS_PASSWORD: {{ .Values.secret.REDIS_PASSWORD | b64enc | quote }} + {{- range $k, $v := .Values.secret.extra }} + {{ $k }}: {{ $v | b64enc | quote }} + {{- end }} diff --git a/examples/kubernetes/firecrawl-helm/templates/worker-deployment.yaml b/examples/kubernetes/firecrawl-helm/templates/worker-deployment.yaml index 787dcfe4a8..64aaa57226 100644 --- a/examples/kubernetes/firecrawl-helm/templates/worker-deployment.yaml +++ b/examples/kubernetes/firecrawl-helm/templates/worker-deployment.yaml @@ -5,7 +5,7 @@ metadata: labels: app: {{ include "firecrawl.name" . }}-worker spec: - replicas: {{ .Values.replicaCount }} + replicas: {{ .Values.worker.replicaCount | default .Values.replicaCount }} selector: matchLabels: app: {{ include "firecrawl.name" . }}-worker @@ -18,16 +18,39 @@ spec: imagePullSecrets: {{- toYaml .Values.imagePullSecrets | nindent 8 }} {{- end }} + terminationGracePeriodSeconds: 60 containers: - name: worker image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - args: [ "pnpm", "run", "workers" ] + command: [ "node" ] + args: [ "--max-old-space-size=3072", "dist/src/services/queue-worker.js" ] + ports: + - containerPort: {{ .Values.worker.port | int }} env: - name: FLY_PROCESS_GROUP value: "worker" + - name: WORKER_PORT + value: {{ .Values.worker.port | quote }} + - name: NUQ_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name envFrom: - configMapRef: name: {{ include "firecrawl.fullname" . }}-config - secretRef: name: {{ include "firecrawl.fullname" . }}-secret + {{- if and .Values.resources.enabled .Values.worker.resources }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /liveness + port: {{ .Values.worker.port | int }} + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 diff --git a/examples/kubernetes/firecrawl-helm/values.yaml b/examples/kubernetes/firecrawl-helm/values.yaml index 853462b3d5..afa9868fb4 100644 --- a/examples/kubernetes/firecrawl-helm/values.yaml +++ b/examples/kubernetes/firecrawl-helm/values.yaml @@ -1,7 +1,44 @@ replicaCount: 1 +resources: + enabled: false + +api: + replicaCount: 1 + resources: + requests: + memory: "4G" + cpu: "2000m" + limits: + memory: "6G" + cpu: "2000m" + +worker: + replicaCount: 1 + port: 3005 + resources: + requests: + memory: "3G" + cpu: "1000m" + limits: + memory: "4G" + cpu: "1000m" + +extractWorker: + enabled: true + replicaCount: 1 + port: 3004 + resources: + requests: + memory: "2G" + cpu: "1000m" + limits: + memory: "3G" + cpu: "1000m" + nuqWorker: replicaCount: 5 + port: 3006 resources: requests: memory: "3G" @@ -10,14 +47,27 @@ nuqWorker: memory: "4G" cpu: "1000m" +nuqPrefetchWorker: + enabled: true + replicaCount: 1 + port: 3011 + resources: + requests: + memory: "1G" + cpu: "500m" + limits: + memory: "2G" + cpu: "1000m" + nuqPostgres: replicaCount: 1 image: - repository: ghcr.io/firecrawl/nuq-postgres + repository: docker.io/winkkgmbh/nuq-postgres tag: latest pullPolicy: Always auth: username: postgres + password: password database: postgres resources: requests: @@ -28,17 +78,32 @@ nuqPostgres: cpu: "500m" persistence: enabled: false + size: 10Gi + storageClass: "" + +rabbitmq: + enabled: true + image: rabbitmq:3-management + replicaCount: 1 + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "1Gi" + cpu: "500m" image: - repository: ghcr.io/firecrawl/firecrawl + repository: docker.io/winkkgmbh/firecrawl tag: "latest" pullPolicy: Always - dockerSecretEnabled: true + dockerSecretEnabled: false playwright: - repository: ghcr.io/firecrawl/playwright-service + repository: docker.io/winkkgmbh/firecrawl-playwright tag: "latest" pullPolicy: Always + replicaCount: 1 redis: image: redis:alpine @@ -54,20 +119,49 @@ service: redis: type: ClusterIP port: 6379 + rabbitmq: + type: ClusterIP + port: 5672 + managementPort: 15672 config: NUM_WORKERS_PER_QUEUE: "8" PORT: "3002" HOST: "0.0.0.0" - REDIS_URL: "redis://redis:6379" - REDIS_RATE_LIMIT_URL: "redis://redis:6379" - PLAYWRIGHT_MICROSERVICE_URL: "http://playwright-service:3000" + WORKER_PORT: "3005" + EXTRACT_WORKER_PORT: "3004" + NUQ_WORKER_PORT: "3006" + NUQ_PREFETCH_WORKER_PORT: "3011" + NUQ_WORKER_COUNT: "5" USE_DB_AUTHENTICATION: "false" - HDX_NODE_BETA_MODE: "1" - NUQ_DATABASE_URL: "postgresql://postgres:password@nuq-postgres:5432/postgres" + IS_KUBERNETES: "true" + ENV: "production" + LOGGING_LEVEL: "INFO" + FIRECRAWL_APP_SCHEME: "http" + FIRECRAWL_APP_HOST: "" + FIRECRAWL_APP_PORT: "" + REDIS_URL: "" + REDIS_RATE_LIMIT_URL: "" + PLAYWRIGHT_MICROSERVICE_URL: "" + NUQ_DATABASE_URL: "" + NUQ_DATABASE_URL_LISTEN: "" + NUQ_RABBITMQ_URL: "" + OPENAI_BASE_URL: "" + MODEL_NAME: "" + MODEL_EMBEDDING_NAME: "" + OLLAMA_BASE_URL: "" + PROXY_SERVER: "" + PROXY_USERNAME: "" + SEARXNG_ENDPOINT: "" + SEARXNG_ENGINES: "" + SEARXNG_CATEGORIES: "" + SELF_HOSTED_WEBHOOK_URL: "" + extra: {} playwrightConfig: PORT: "3000" + BLOCK_MEDIA: "" + MAX_CONCURRENT_PAGES: "10" secret: OPENAI_API_KEY: "" @@ -75,11 +169,14 @@ secret: LLAMAPARSE_API_KEY: "" BULL_AUTH_KEY: "" TEST_API_KEY: "" - SCRAPING_BEE_API_KEY: "" - STRIPE_PRICE_ID_STANDARD: "" - STRIPE_PRICE_ID_SCALE: "" FIRE_ENGINE_BETA_URL: "" + SUPABASE_ANON_TOKEN: "" + SUPABASE_URL: "" + SUPABASE_SERVICE_TOKEN: "" + PROXY_PASSWORD: "" + SELF_HOSTED_WEBHOOK_HMAC_SECRET: "" REDIS_PASSWORD: "" + extra: {} imagePullSecrets: - name: docker-registry-secret