diff --git a/.github/workflows/check-queues.yml b/.github/archive/check-queues.yml similarity index 93% rename from .github/workflows/check-queues.yml rename to .github/archive/check-queues.yml index 5cb5d9ca7a..b0fde857c2 100644 --- a/.github/workflows/check-queues.yml +++ b/.github/archive/check-queues.yml @@ -1,7 +1,5 @@ name: Check Queues -on: - schedule: - - cron: '*/5 * * * *' +on: [] env: BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} diff --git a/.github/workflows/clean-before-24h-complete-jobs.yml b/.github/archive/clean-before-24h-complete-jobs.yml similarity index 93% rename from .github/workflows/clean-before-24h-complete-jobs.yml rename to .github/archive/clean-before-24h-complete-jobs.yml index d897a19add..e2318f5eb1 100644 --- a/.github/workflows/clean-before-24h-complete-jobs.yml +++ b/.github/archive/clean-before-24h-complete-jobs.yml @@ -1,7 +1,5 @@ name: Clean Every 30 Minutes Before 24h Completed Jobs -on: - schedule: - - cron: '30 * * * *' +on: [] env: BULL_AUTH_KEY: ${{ secrets.BULL_AUTH_KEY }} diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts index d1e031edf3..ba234c69c0 100644 --- a/apps/api/src/controllers/v0/admin/queue.ts +++ b/apps/api/src/controllers/v0/admin/queue.ts @@ -1,68 +1,7 @@ import { Request, Response } from "express"; -import { Job } from "bullmq"; import { Logger } from "../../../lib/logger"; import { getScrapeQueue } from "../../../services/queue-service"; -import { checkAlerts } from "../../../services/alerts"; - -export async function cleanBefore24hCompleteJobsController( - req: Request, - res: Response -) { - Logger.info("🐂 Cleaning jobs older than 24h"); - try { - const scrapeQueue = getScrapeQueue(); - const batchSize = 10; - const numberOfBatches = 9; // Adjust based on your needs - const completedJobsPromises: Promise[] = []; - for (let i = 0; i < numberOfBatches; i++) { - completedJobsPromises.push( - scrapeQueue.getJobs( - ["completed"], - i * batchSize, - i * batchSize + batchSize, - true - ) - ); - } - const completedJobs: Job[] = ( - await Promise.all(completedJobsPromises) - ).flat(); - const before24hJobs = - completedJobs.filter( - (job) => job.finishedOn < Date.now() - 24 * 60 * 60 * 1000 - ) || []; - - let count = 0; - - if (!before24hJobs) { - return res.status(200).send(`No jobs to remove.`); - } - - for (const job of before24hJobs) { - try { - await job.remove(); - count++; - } catch (jobError) { - Logger.error(`🐂 Failed to remove job with ID ${job.id}: ${jobError}`); - } - } - return res.status(200).send(`Removed ${count} completed jobs.`); - } catch (error) { - Logger.error(`🐂 Failed to clean last 24h complete jobs: ${error}`); - return res.status(500).send("Failed to clean jobs"); - } -} - -export async function checkQueuesController(req: Request, res: Response) { - try { - await checkAlerts(); - return res.status(200).send("Alerts initialized"); - } catch (error) { - Logger.debug(`Failed to initialize alerts: ${error}`); - return res.status(500).send("Failed to initialize alerts"); - } -} // Use this as a "health check" that way we dont destroy the server export async function queuesController(req: Request, res: Response) { diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts index 9a2e265a7c..2c24853d68 100644 --- a/apps/api/src/routes/admin.ts +++ b/apps/api/src/routes/admin.ts @@ -1,10 +1,6 @@ import express from "express"; import { redisHealthController } from "../controllers/v0/admin/redis-health"; -import { - checkQueuesController, - cleanBefore24hCompleteJobsController, - queuesController, -} from "../controllers/v0/admin/queue"; +import { queuesController } from "../controllers/v0/admin/queue"; export const adminRouter = express.Router(); @@ -13,14 +9,4 @@ adminRouter.get( redisHealthController ); -adminRouter.get( - `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`, - cleanBefore24hCompleteJobsController -); - -adminRouter.get( - `/admin/${process.env.BULL_AUTH_KEY}/check-queues`, - checkQueuesController -); - adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController); diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts deleted file mode 100644 index 7707b658bd..0000000000 --- a/apps/api/src/services/alerts/index.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { Logger } from "../../../src/lib/logger"; -import { getScrapeQueue } from "../queue-service"; - -export async function checkAlerts() { - try { - if ( - process.env.ENV === "production" && - process.env.ALERT_NUM_ACTIVE_JOBS && - process.env.ALERT_NUM_WAITING_JOBS - ) { - Logger.info("Initializing alerts"); - const checkActiveJobs = async () => { - try { - const scrapeQueue = getScrapeQueue(); - const activeJobs = await scrapeQueue.getActiveCount(); - if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) { - Logger.warn( - `Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.` - ); - } else { - Logger.info( - `Number of active jobs is under ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}` - ); - } - } catch (error) { - Logger.error(`Failed to check active jobs: ${error}`); - } - }; - - const checkWaitingQueue = async () => { - const scrapeQueue = getScrapeQueue(); - const waitingJobs = await scrapeQueue.getWaitingCount(); - - if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) { - Logger.warn( - `Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.` - ); - } - }; - - const checkAll = async () => { - await checkActiveJobs(); - await checkWaitingQueue(); - }; - - await checkAll(); - // setInterval(checkAll, 10000); // Run every - } - } catch (error) { - Logger.error(`Failed to initialize alerts: ${error}`); - } -}