From ccca484d23000e92908308cd86cc152067d45876 Mon Sep 17 00:00:00 2001 From: colin Date: Sat, 18 Jul 2026 05:35:57 +0700 Subject: [PATCH] ci: disable stale cron workflows and remove their matching controllers (#50) Two scheduled cron workflows (check-queues every 5 min, clean-before-24h-complete-jobs every 30 min) have been failing on every run since BULL_AUTH_KEY is not configured as a repo secret AND the curl target api.firecrawl.dev is upstream Mendable's SaaS, not this fork's deployment -- the workflows were copied verbatim when the fork diverged and never adapted. See issue #50 for run-log receipts. Even if these were re-pointed at a fork-owned deployment with the secret set, the operational benefit would be nil: check-queues calls checkAlerts() which is a no-op by default (guarded by ENV=production AND ALERT_NUM_ACTIVE_JOBS AND ALERT_NUM_WAITING_JOBS -- none plumbed in .env.example, compose, or docs; Slack output was stripped in b024f08); clean-before-24h-complete-jobs is largely redundant with BullMQ's removeOnComplete.age=90000 (25h) auto-eviction. Upstream firecrawl/firecrawl has fully removed both workflows AND the matching controllers/v0/admin/queue.ts and services/alerts.ts. Changes: - Move .github/workflows/check-queues.yml and clean-before-24h-complete-jobs.yml into .github/archive/ with on: [] (matches the existing archive convention for the 6 SDK workflows). - Remove checkQueuesController and cleanBefore24hCompleteJobsController from controllers/v0/admin/queue.ts and their route registrations from routes/admin.ts (the only callers of those endpoints were the disabled crons). Keep queuesController (JSON health endpoint, still useful to self-hosters per SELF_HOST.md:86) and redisHealthController (upstream also kept it). - Delete the now-dead apps/api/src/services/alerts/ directory (its only consumer was checkQueuesController). Unchanged: BULL_AUTH_KEY stays in apps/api/.env.example and docker-compose.yaml for self-hosters to protect their own admin endpoints to their own deployments. README.md, CONTRIBUTING.md, and SELF_HOST.md untouched -- the stale SELF_HOST.md:86 'Bull Queue Manager UI' reference is tracked separately in issue #51. A latent admin-guard bug in admin.ts (routes mount unconditionally at /admin/undefined/... when BULL_AUTH_KEY unset; upstream gates behind if (config.BULL_AUTH_KEY)) is flagged in the PR body for a separate follow-up. Verified: pnpm run build (tsc) passes clean with no orphan-import errors. pnpm test runs -- 7 of 11 unit suites pass (74/89 tests); the 4 failing suites (parseTable, urlValidation, html-to-markdown, crawl) are pre-existing environmental issues (Cheerio load, native Go html-to-markdown module, services/idempotency/validate missing, test-expectation drift) unrelated to anything removed here. No test file references any of the removed symbols (grep across *.test.ts). --- .../{workflows => archive}/check-queues.yml | 4 +- .../clean-before-24h-complete-jobs.yml | 4 +- apps/api/src/controllers/v0/admin/queue.ts | 61 ------------------- apps/api/src/routes/admin.ts | 16 +---- apps/api/src/services/alerts/index.ts | 52 ---------------- 5 files changed, 3 insertions(+), 134 deletions(-) rename .github/{workflows => archive}/check-queues.yml (93%) rename .github/{workflows => archive}/clean-before-24h-complete-jobs.yml (93%) delete mode 100644 apps/api/src/services/alerts/index.ts diff --git a/.github/workflows/check-queues.yml b/.github/archive/check-queues.yml similarity index 93% rename from .github/workflows/check-queues.yml rename to .github/archive/check-queues.yml index 5cb5d9ca7a..b0fde857c2 100644 --- a/.github/workflows/check-queues.yml +++ b/.github/archive/check-queues.yml @@ -1,7 +1,5 @@ name: Check Queues -on: - schedule: - - cron: '*/5 * * * *' +on: [] env: BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} diff --git a/.github/workflows/clean-before-24h-complete-jobs.yml b/.github/archive/clean-before-24h-complete-jobs.yml similarity index 93% rename from .github/workflows/clean-before-24h-complete-jobs.yml rename to .github/archive/clean-before-24h-complete-jobs.yml index d897a19add..e2318f5eb1 100644 --- a/.github/workflows/clean-before-24h-complete-jobs.yml +++ b/.github/archive/clean-before-24h-complete-jobs.yml @@ -1,7 +1,5 @@ name: Clean Every 30 Minutes Before 24h Completed Jobs -on: - schedule: - - cron: '30 * * * *' +on: [] env: BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts index d1e031edf3..ba234c69c0 100644 --- a/apps/api/src/controllers/v0/admin/queue.ts +++ b/apps/api/src/controllers/v0/admin/queue.ts @@ -1,68 +1,7 @@ import { Request, Response } from "express"; -import { Job } from "bullmq"; import { Logger } from "../../../lib/logger"; import { getScrapeQueue } from "../../../services/queue-service"; -import { checkAlerts } from "../../../services/alerts"; - -export async function cleanBefore24hCompleteJobsController( - req: Request, - res: Response -) { - Logger.info("🐂 Cleaning jobs older than 24h"); - try { - const scrapeQueue = getScrapeQueue(); - const batchSize = 10; - const numberOfBatches = 9; // Adjust based on your needs - const completedJobsPromises: Promise[] = []; - for (let i = 0; i < numberOfBatches; i++) { - completedJobsPromises.push( - scrapeQueue.getJobs( - ["completed"], - i * batchSize, - i * batchSize + batchSize, - true - ) - ); - } - const completedJobs: Job[] = ( - await Promise.all(completedJobsPromises) - ).flat(); - const before24hJobs = - completedJobs.filter( - (job) => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000 - ) || []; - - let count = 0; - - if (!before24hJobs) { - return res.status(200).send(`No jobs to remove.`); - } - - for (const job of before24hJobs) { - try { - await job.remove(); - count++; - } catch (jobError) { - Logger.error(`🐂 Failed to remove job with ID ${job.id}: ${jobError}`); - } - } - return res.status(200).send(`Removed ${count} completed jobs.`); - } catch (error) { - Logger.error(`🐂 Failed to clean last 24h complete jobs: ${error}`); - return res.status(500).send("Failed to clean jobs"); - } -} - -export async function checkQueuesController(req: Request, res: Response) { - try { - await checkAlerts(); - return res.status(200).send("Alerts initialized"); - } catch (error) { - Logger.debug(`Failed to initialize alerts: ${error}`); - return res.status(500).send("Failed to initialize alerts"); - } -} // Use this as a "health check" that way we dont destroy the server export async function queuesController(req: Request, res: Response) { diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts index 9a2e265a7c..2c24853d68 100644 --- a/apps/api/src/routes/admin.ts +++ b/apps/api/src/routes/admin.ts @@ -1,10 +1,6 @@ import express from "express"; import { redisHealthController } from "../controllers/v0/admin/redis-health"; -import { - checkQueuesController, - cleanBefore24hCompleteJobsController, - queuesController, -} from "../controllers/v0/admin/queue"; +import { queuesController } from "../controllers/v0/admin/queue"; export const adminRouter = express.Router(); @@ -13,14 +9,4 @@ adminRouter.get( redisHealthController ); -adminRouter.get( - `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, - cleanBefore24hCompleteJobsController -); - -adminRouter.get( - `/admin/${process.env.BULL_AUTH_KEY}/check-queues`, - checkQueuesController -); - adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController); diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts deleted file mode 100644 index 7707b658bd..0000000000 --- a/apps/api/src/services/alerts/index.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { Logger } from "../../../src/lib/logger"; -import { getScrapeQueue } from "../queue-service"; - -export async function checkAlerts() { - try { - if ( - process.env.ENV === "production" && - process.env.ALERT_NUM_ACTIVE_JOBS && - process.env.ALERT_NUM_WAITING_JOBS - ) { - Logger.info("Initializing alerts"); - const checkActiveJobs = async () => { - try { - const scrapeQueue = getScrapeQueue(); - const activeJobs = await scrapeQueue.getActiveCount(); - if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) { - Logger.warn( - `Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.` - ); - } else { - Logger.info( - `Number of active jobs is under ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}` - ); - } - } catch (error) { - Logger.error(`Failed to check active jobs: ${error}`); - } - }; - - const checkWaitingQueue = async () => { - const scrapeQueue = getScrapeQueue(); - const waitingJobs = await scrapeQueue.getWaitingCount(); - - if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) { - Logger.warn( - `Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.` - ); - } - }; - - const checkAll = async () => { - await checkActiveJobs(); - await checkWaitingQueue(); - }; - - await checkAll(); - // setInterval(checkAll, 10000); // Run every - } - } catch (error) { - Logger.error(`Failed to initialize alerts: ${error}`); - } -}