From cf6a5f4ccf6215b08709a4e31c936761364cfbc0 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Mon, 6 Jul 2026 14:21:59 +0000 Subject: [PATCH 1/5] feat(api): migrate from Celery to RQ and expose queue position Replace Celery with RQ to allow reporting pending task position and in-progress count in task status responses, resolving #107. Co-authored-by: Cursor --- bases/ecoindex/backend/routers/tasks.py | 96 +++-- bases/ecoindex/worker/health.py | 21 +- bases/ecoindex/worker/tasks.py | 42 +- components/ecoindex/config/settings.py | 4 + components/ecoindex/models/tasks.py | 9 + .../ecoindex/worker_component/__init__.py | 115 +++++- projects/ecoindex_api/.env.template | 6 +- projects/ecoindex_api/README.md | 17 +- projects/ecoindex_api/Taskfile.yml | 49 ++- .../ecoindex_api/docker-compose.yml.template | 9 +- .../ecoindex_api/docker/worker/dockerfile | 3 +- .../ecoindex_api/docker/worker/entrypoint.sh | 2 +- projects/ecoindex_api/openapi.json | 365 ++++++++++++------ projects/ecoindex_api/pyproject.toml | 3 +- .../ecoindex_api/scripts/start_rq_workers.sh | 23 ++ projects/ecoindex_api/scripts/wait_redis.py | 70 ++++ pyproject.toml | 2 +- uv.lock | 278 ++++++------- 18 files changed, 740 insertions(+), 374 deletions(-) create mode 100755 projects/ecoindex_api/scripts/start_rq_workers.sh create mode 100644 projects/ecoindex_api/scripts/wait_redis.py diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index 4ba1944..62e18c8 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -1,10 +1,8 @@ -from json import loads from typing import Annotated from urllib.parse import urlparse, urlunparse import idna import requests -from celery.result import AsyncResult from ecoindex.backend.dependencies.validation import validate_api_key_batch from ecoindex.backend.models.dependencies_parameters.id import IdParameter from ecoindex.backend.utils import check_quota @@ -17,10 +15,22 @@ example_daily_limit_response, example_host_unreachable, ) -from ecoindex.models.tasks import QueueTaskApi, QueueTaskApiBatch, QueueTaskResult +from ecoindex.models.tasks import QueueTaskApi, QueueTaskApiBatch from ecoindex.scraper.scrap import EcoindexScraper from ecoindex.worker.tasks import ecoindex_batch_import_task, ecoindex_task -from ecoindex.worker_component import app as task_app +from ecoindex.worker_component import ( + NoSuchJobError, + cancel_job, + ecoindex_batch_queue, + ecoindex_queue, + fetch_job, + get_queue_for_job, + get_queue_position, + get_retry_policy, + get_tasks_in_progress, + map_job_status_to_task_status, + parse_job_result, +) from fastapi import APIRouter, Depends, HTTPException, Response, status from fastapi.params import Body from sqlmodel.ext.asyncio.session import AsyncSession @@ -70,6 +80,18 @@ def convert_url_to_punycode(url: str) -> str: return url +def _enqueue_settings(*, with_retry: bool = True) -> dict: + settings = Settings() + enqueue_settings = { + "result_ttl": settings.RQ_RESULT_TTL, + "failure_ttl": settings.RQ_FAILURE_TTL, + "job_timeout": settings.RQ_JOB_TIMEOUT, + } + if with_retry: + enqueue_settings["retry"] = get_retry_policy() + return enqueue_settings + + @router.post( name="Add new ecoindex analysis task to the waiting queue", path="/", @@ -142,14 +164,16 @@ async def add_ecoindex_analysis_task( detail=f"The URL {web_page.url} is unreachable. Are you really sure of this url? 🤔 ({e.response.status_code if e.response else ''})", ) - task_result = ecoindex_task.delay( # type: ignore + job = ecoindex_queue.enqueue( + ecoindex_task, url=str(web_page.url), width=web_page.width, height=web_page.height, custom_headers=headers, + **_enqueue_settings(), ) - return task_result.id + return job.id @router.get( @@ -166,23 +190,35 @@ async def get_ecoindex_analysis_task_by_id( response: Response, id: IdParameter, ) -> QueueTaskApi: - t = AsyncResult(id=str(id), app=task_app) + try: + job = fetch_job(str(id)) + except NoSuchJobError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Task not found", + ) from exc + + queue = get_queue_for_job(job) + task_status = map_job_status_to_task_status(job) + result = parse_job_result(job) task_response = QueueTaskApi( - id=str(t.id), - status=t.state, + id=job.id, + status=task_status, + queue_position=get_queue_position(job, queue), + tasks_in_progress=get_tasks_in_progress(queue), ) - if t.state == TaskStatus.PENDING: + if task_status == TaskStatus.PENDING: response.status_code = status.HTTP_425_TOO_EARLY return task_response - if t.state == TaskStatus.SUCCESS: - task_response.ecoindex_result = QueueTaskResult(**loads(t.result)) + if task_status == TaskStatus.SUCCESS: + task_response.ecoindex_result = result - if t.state == TaskStatus.FAILURE: - task_response.task_error = t.info + if task_status == TaskStatus.FAILURE: + task_response.task_error = result response.status_code = status.HTTP_200_OK @@ -198,9 +234,13 @@ async def get_ecoindex_analysis_task_by_id( async def delete_ecoindex_analysis_task_by_id( id: IdParameter, ) -> None: - res = task_app.control.revoke(id, terminate=True, signal="SIGKILL") - - return res + try: + cancel_job(str(id)) + except NoSuchJobError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Task not found", + ) from exc @router.post( @@ -227,12 +267,14 @@ async def add_ecoindex_analysis_task_batch( ], batch_key: str = Depends(validate_api_key_batch), ): - task_result = ecoindex_batch_import_task.delay( # type: ignore + job = ecoindex_batch_queue.enqueue( + ecoindex_batch_import_task, results=[result.model_dump() for result in results], source=batch_key["source"], # type: ignore + **_enqueue_settings(with_retry=False), ) - return task_result.id + return job.id @router.get( @@ -250,14 +292,22 @@ async def get_ecoindex_analysis_batch_task_by_id( id: IdParameter, _: str = Depends(validate_api_key_batch), ) -> QueueTaskApiBatch: - t = AsyncResult(id=str(id), app=task_app) + try: + job = fetch_job(str(id)) + except NoSuchJobError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Task not found", + ) from exc + + task_status = map_job_status_to_task_status(job) task_response = QueueTaskApiBatch( - id=str(t.id), - status=t.state, + id=job.id, + status=task_status, ) - if t.state == TaskStatus.PENDING: + if task_status == TaskStatus.PENDING: response.status_code = status.HTTP_425_TOO_EARLY return task_response diff --git a/bases/ecoindex/worker/health.py b/bases/ecoindex/worker/health.py index b753bce..ca3eba3 100644 --- a/bases/ecoindex/worker/health.py +++ b/bases/ecoindex/worker/health.py @@ -1,18 +1,23 @@ from ecoindex.models.api import HealthWorker, HealthWorkers -from ecoindex.worker.tasks import app +from ecoindex.worker_component import redis_connection +from redis.exceptions import ConnectionError as RedisConnectionError +from rq.worker import Worker def is_worker_healthy() -> HealthWorkers: - workers = [] - workers_ping = app.control.ping() - - for worker in workers_ping: - for name in worker: + try: + workers = [] + for worker in Worker.all(connection=redis_connection): workers.append( - HealthWorker(name=name, healthy=True if "ok" in worker[name] else False) + HealthWorker( + name=worker.name, + healthy=worker.state in ("busy", "idle"), + ) ) + except RedisConnectionError: + return HealthWorkers(healthy=False, workers=[]) return HealthWorkers( - healthy=False if False in [w.healthy for w in workers] or not workers else True, + healthy=bool(workers) and all(w.healthy for w in workers), workers=workers, ) diff --git a/bases/ecoindex/worker/tasks.py b/bases/ecoindex/worker/tasks.py index a90431a..9c61865 100644 --- a/bases/ecoindex/worker/tasks.py +++ b/bases/ecoindex/worker/tasks.py @@ -1,6 +1,7 @@ from asyncio import run from os import getcwd from urllib.parse import urlparse +from uuid import UUID from ecoindex.backend.utils import check_quota, format_exception_response from ecoindex.config.settings import Settings @@ -19,30 +20,31 @@ from ecoindex.models.enums import TaskStatus from ecoindex.models.tasks import QueueTaskError, QueueTaskResult from ecoindex.scraper.scrap import EcoindexScraper -from ecoindex.worker_component import app from playwright._impl._errors import Error as WebDriverException +from rq import get_current_job from sentry_sdk import init as sentry_init if Settings().GLITCHTIP_DSN: sentry_init(Settings().GLITCHTIP_DSN) -@app.task( - name="ecoindex.analysis", - bind=True, - autoretry_for=(Exception,), - retry_backoff=5, - retry_kwargs={"max_retries": 5}, - timezone=Settings().TZ, - queue="ecoindex", - dont_autoretry_for=[EcoindexScraperStatusException, TypeError], -) +def _get_task_id() -> UUID: + job = get_current_job() + if job is None: + raise RuntimeError("No RQ job context available") + return UUID(job.id) + + def ecoindex_task( - self, url: str, width: int, height: int, custom_headers: dict[str, str] + url: str, width: int, height: int, custom_headers: dict[str, str] ) -> str: queue_task_result = run( async_ecoindex_task( - self, url=url, width=width, height=height, custom_headers=custom_headers + task_id=_get_task_id(), + url=url, + width=width, + height=height, + custom_headers=custom_headers, ) ) @@ -50,7 +52,7 @@ def ecoindex_task( async def async_ecoindex_task( - self, + task_id: UUID, url: str, width: int, height: int, @@ -68,7 +70,7 @@ async def async_ecoindex_task( wait_after_scroll=Settings().WAIT_AFTER_SCROLL, wait_before_scroll=Settings().WAIT_BEFORE_SCROLL, screenshot=ScreenShot( - id=str(self.request.id), folder=f"{getcwd()}/screenshots/v1" + id=str(task_id), folder=f"{getcwd()}/screenshots/v1" ) if Settings().ENABLE_SCREENSHOT else None, @@ -79,7 +81,7 @@ async def async_ecoindex_task( db_result = await save_ecoindex_result_db( session=session, - id=self.request.id, + id=task_id, ecoindex_result=ecoindex, ) @@ -164,13 +166,7 @@ async def async_ecoindex_task( ) -@app.task( - name="ecoindex.batch_import", - timezone=Settings().TZ, - queue="ecoindex_batch", - bind=True, -) -def ecoindex_batch_import_task(self, results: list[dict], source: str): +def ecoindex_batch_import_task(results: list[dict], source: str) -> str: queue_task_result = run( async_ecoindex_batch_import_task( results=[ApiEcoindex.model_validate(result) for result in results], diff --git a/components/ecoindex/config/settings.py b/components/ecoindex/config/settings.py index 3989c77..5dd52c6 100644 --- a/components/ecoindex/config/settings.py +++ b/components/ecoindex/config/settings.py @@ -20,6 +20,10 @@ class Settings(BaseSettings): FRONTEND_BASE_URL: str = "https://www.ecoindex.fr" GLITCHTIP_DSN: str = "" REDIS_CACHE_HOST: str = "localhost" + RQ_FAILURE_TTL: int = 86400 + RQ_JOB_TIMEOUT: int = 600 + RQ_RESULT_TTL: int = 86400 + RQ_WORKERS: int = 3 SCREENSHOTS_GID: int | None = None SCREENSHOTS_UID: int | None = None TZ: str = "Europe/Paris" diff --git a/components/ecoindex/models/tasks.py b/components/ecoindex/models/tasks.py index 7f92f37..ae20bfe 100644 --- a/components/ecoindex/models/tasks.py +++ b/components/ecoindex/models/tasks.py @@ -46,6 +46,15 @@ class QueueTaskApi(BaseModel): default=..., title="Status of the current task. Can be PENDING, FAILURE, SUCCESS", ) + queue_position: int | None = Field( + default=None, + title="Position of the task in the waiting queue (0 = next to run)", + description="Null when the task is running or already completed.", + ) + tasks_in_progress: int = Field( + default=0, + title="Number of tasks currently being processed on this queue", + ) ecoindex_result: QueueTaskResult | None = Field( default=None, title="Result of the Ecoindex analysis" ) diff --git a/components/ecoindex/worker_component/__init__.py b/components/ecoindex/worker_component/__init__.py index 780c673..482c48f 100644 --- a/components/ecoindex/worker_component/__init__.py +++ b/components/ecoindex/worker_component/__init__.py @@ -1,11 +1,110 @@ -from celery import Celery +from json import loads +from typing import Any + from ecoindex.config.settings import Settings +from ecoindex.models.enums import TaskStatus +from ecoindex.models.tasks import QueueTaskResult +from redis import Redis +from rq import Queue, Retry +from rq.exceptions import NoSuchJobError +from rq.job import Job +from rq.registry import StartedJobRegistry + +ECOINDEX_QUEUE_NAME = "ecoindex" +ECOINDEX_BATCH_QUEUE_NAME = "ecoindex_batch" + +_settings = Settings() + + +def get_redis_connection() -> Redis: + return Redis(host=_settings.REDIS_CACHE_HOST, port=6379, db=0) + + +redis_connection: Redis = get_redis_connection() -app: Celery = Celery( - "tasks", - broker=f"redis://{Settings().REDIS_CACHE_HOST}:6379/0", - backend=f"redis://{Settings().REDIS_CACHE_HOST}:6379/1", - broker_connection_retry=True, - broker_connection_retry_on_startup=True, - broker_connection_max_retries=10, +ecoindex_queue: Queue = Queue( + ECOINDEX_QUEUE_NAME, + connection=redis_connection, + default_result_ttl=_settings.RQ_RESULT_TTL, ) +ecoindex_batch_queue: Queue = Queue( + ECOINDEX_BATCH_QUEUE_NAME, + connection=redis_connection, + default_result_ttl=_settings.RQ_RESULT_TTL, +) + +_queues_by_name: dict[str, Queue] = { + ECOINDEX_QUEUE_NAME: ecoindex_queue, + ECOINDEX_BATCH_QUEUE_NAME: ecoindex_batch_queue, +} + + +def get_retry_policy() -> Retry: + return Retry(max=5, interval=[5, 10, 20, 40, 80]) + + +def get_queue_for_job(job: Job) -> Queue: + return _queues_by_name.get(job.origin, ecoindex_queue) + + +def get_tasks_in_progress(queue: Queue) -> int: + return StartedJobRegistry(queue.name, connection=queue.connection).count + + +def fetch_job(job_id: str) -> Job: + return Job.fetch(job_id, connection=redis_connection) + + +def map_job_status_to_task_status(job: Job) -> TaskStatus: + status = job.get_status() + if status == "finished": + return TaskStatus.SUCCESS + if status == "failed": + return TaskStatus.FAILURE + return TaskStatus.PENDING + + +def get_queue_position(job: Job, queue: Queue) -> int | None: + if job.get_status() != "queued": + return None + return queue.get_job_position(job) + + +def parse_job_result(job: Job) -> QueueTaskResult | Any | None: + if job.get_status() == "finished": + result = job.result + if isinstance(result, str): + return QueueTaskResult(**loads(result)) + return result + if job.get_status() == "failed": + return job.exc_info + return None + + +def cancel_job(job_id: str) -> None: + from rq.command import send_stop_job_command + + job = fetch_job(job_id) + status = job.get_status() + if status == "queued": + job.cancel() + elif status == "started": + send_stop_job_command(redis_connection, job.id) + + +__all__ = [ + "ECOINDEX_BATCH_QUEUE_NAME", + "ECOINDEX_QUEUE_NAME", + "NoSuchJobError", + "cancel_job", + "ecoindex_batch_queue", + "ecoindex_queue", + "fetch_job", + "get_queue_for_job", + "get_queue_position", + "get_retry_policy", + "get_tasks_in_progress", + "map_job_status_to_task_status", + "parse_job_result", + "redis_connection", +] diff --git a/projects/ecoindex_api/.env.template b/projects/ecoindex_api/.env.template index 37128eb..cf2b84c 100644 --- a/projects/ecoindex_api/.env.template +++ b/projects/ecoindex_api/.env.template @@ -9,10 +9,12 @@ # DEBUG=1 # ENABLE_SCREENSHOT=1 # EXCLUDED_HOSTS='["localhost","127.0.0.1"]' -# FLOWER_BASIC_AUTH=ecoindex:password -# FLOWER_PORT=5555 # GLITCHTIP_DSN= # REDIS_CACHE_HOST=redis +# RQ_FAILURE_TTL=86400 +# RQ_JOB_TIMEOUT=600 +# RQ_RESULT_TTL=86400 +# RQ_WORKERS=3 # SCREENSHOTS_GID=1006 # SCREENSHOTS_UID=1006 # TZ=Europe/Paris diff --git a/projects/ecoindex_api/README.md b/projects/ecoindex_api/README.md index dd73eb0..1739a66 100644 --- a/projects/ecoindex_api/README.md +++ b/projects/ecoindex_api/README.md @@ -15,7 +15,7 @@ This tool provides an easy way to analyze websites with [Ecoindex](https://www.e - Limit the number of request per day for a given host - Get screenshots of the analyzed page -This API is built on top of [ecoindex-scraper](https://pypi.org/project/ecoindex-scraper/) with [FastAPI](https://fastapi.tiangolo.com/) and [Celery](https://docs.celeryq.dev/) +This API is built on top of [ecoindex-scraper](https://pypi.org/project/ecoindex-scraper/) with [FastAPI](https://fastapi.tiangolo.com/) and [RQ](https://python-rq.org/) ## OpenAPI specification @@ -28,13 +28,12 @@ The API specification can be found in the [documentation](projects/ecoindex_api/ ## Installation -With this docker setup you get 6 services running that are enough to make it all work: +With this docker setup you get 4 services running that are enough to make it all work: - `db`: A MySQL instance - `api`: The API instance running FastAPI application -- `worker`: The celery task worker that runs ecoindex analysis -- `redis` (optional): The [redis](https://redis.io/) instance that is used by the Celery worker -- `flower` (optional): The Celery [monitoring interface](https://flower.readthedocs.io/en/latest/) +- `worker`: The RQ task worker that runs ecoindex analysis +- `redis`: The [redis](https://redis.io/) instance that is used by the RQ worker and API cache ### First start @@ -46,7 +45,6 @@ docker compose up -d Every services should start normaly, then you can go to: - [http://localhost:8001/docs](http://localhost:8001/docs) to access to the swagger of the API -- [http://localhost:5555](http://localhost:5555) to access the flower interface (Celery task queue UI. Basic auth is `ecoindex:ecoindex`) ## Configuration @@ -65,12 +63,15 @@ Here are the environment variables you can configure in your `.env` file: | API, Worker | `DAILY_LIMIT_PER_HOST` | 0 | When this variable is set, it won't be possible for a same host to make more request than defined in the same day to avoid overload. If the variable is set, you will get a header `x-remaining-daily-requests: 6` in your response. It is used for the POST methods. If you reach your authorized request quota for the day, the next requests will give you a 429 response. If the variable is set to 0, no limit is set | | API, Worker | `DATABASE_URL` | `sqlite+aiosqlite:///./sql_app.db` | If you run your mysql instance on a dedicated server, you can configure it with your credentials. By default, it uses an sqlite database when running in local | | | API, Worker | `GLITCHTIP_DSN` | `` | If you want to use [Glitchtip](https://glitchtip.com/) to monitor your application, you can set this variable with your DSN. | -| API, Worker, Flower | `REDIS_CACHE_HOST` | `localhost` | The hostname of the redis backend used by Celery but also API to cache results | +| API, Worker | `REDIS_CACHE_HOST` | `localhost` | The hostname of the redis backend used by RQ and API cache | +| API, Worker | `RQ_FAILURE_TTL` | `86400` | Time in seconds before failed job metadata is removed from Redis | +| API, Worker | `RQ_JOB_TIMEOUT` | `600` | Maximum time in seconds a job is allowed to run before being stopped | +| API, Worker | `RQ_RESULT_TTL` | `86400` | Time in seconds before successful job results are removed from Redis | +| Worker | `RQ_WORKERS` | `3` | Number of RQ worker processes started in parallel (one job per process) | | API, Worker | `TZ` | `Europe/Paris` | The timezone used by the API and the worker. | | Worker | `ENABLE_SCREENSHOT` | `False` | If screenshots are enabled, when analyzing the page the image will be generated in the `./screenshot` directory with the image name corresponding to the analysis ID and will be available on the path `/{version}/ecoindexes/{id}/screenshot` | | Worker | `SCREENSHOT_GID` | None | The group used to create the screenshot. If not set, the group of the current user will be used. | | Worker | `SCREENSHOT_UID` | None | The user used to create the screenshot. If not set, the current user will be used. | -| Flower | `FLOWER_BASIC_AUTH` | `ecoindex:ecoindex` | The basic auth used to access the flower interface | ## Local development with [task](https://taskfile.dev) diff --git a/projects/ecoindex_api/Taskfile.yml b/projects/ecoindex_api/Taskfile.yml index 18a8f01..76556d1 100644 --- a/projects/ecoindex_api/Taskfile.yml +++ b/projects/ecoindex_api/Taskfile.yml @@ -12,6 +12,11 @@ vars: PROJECT_NAME: api PACKAGE_NAME: ecoindex_api OUT_DIR: projects/ecoindex_api/dist + RQ_DASHBOARD_PORT: "9181" + RQ_WORKERS: + sh: printf '%s' "${RQ_WORKERS:-3}" + REDIS_DEV_HOST: + sh: uv run python scripts/wait_redis.py tasks: update-openapi: @@ -166,29 +171,52 @@ tasks: start-redis: internal: true cmds: - - docker run --rm -p 6379:6379 -d redis:alpine + - | + if ! uv run python scripts/wait_redis.py > /dev/null 2>&1; then + docker rm -f ecoindex-dev-redis 2>/dev/null || true + docker run -p 6379:6379 -d --name ecoindex-dev-redis redis:alpine 2>/dev/null || docker start ecoindex-dev-redis 2>/dev/null || true + fi + - uv run python scripts/wait_redis.py > /dev/null status: - - docker ps | grep redis + - uv run python scripts/wait_redis.py > /dev/null silent: true start-worker: deps: [start-redis] + env: + REDIS_CACHE_HOST: "{{.REDIS_DEV_HOST}}" + PYTHONPATH: bases:components + RQ_WORKERS: "{{.RQ_WORKERS}}" + RQ_REDIS_URL: redis://{{.REDIS_DEV_HOST}}:6379/0 + USE_UV: "1" cmds: - - uv run watchmedo auto-restart --directory=. --pattern=bases/ecoindex/worker/*.py --recursive -- uv run --package ecoindex_api celery -- -A ecoindex.worker.tasks worker --loglevel=DEBUG --queues=ecoindex,ecoindex_batch -E + - uv run --package ecoindex_api --group dev --group worker watchmedo auto-restart --directory=bases --directory=components --pattern=*.py --recursive -- bash projects/ecoindex_api/scripts/start_rq_workers.sh dir: ../.. silent: true start-backend: + deps: [start-redis] + env: + REDIS_CACHE_HOST: "{{.REDIS_DEV_HOST}}" + PYTHONPATH: bases:components + cmds: + - uv run --package ecoindex_api --group backend uvicorn ecoindex.backend.main:app --host 0.0.0.0 --port 8000 --reload --reload-dir bases --reload-dir components + dir: ../.. + silent: true + + start-rq-dashboard: + desc: Start the RQ dashboard for monitoring queues + deps: [start-redis] cmds: - - uv run uvicorn ecoindex.backend.main:app --host 0.0.0.0 --port 8000 --reload --reload-dir bases --reload-dir components + - uv run --package ecoindex_api --group dev rq-dashboard --bind 0.0.0.0 --port {{.RQ_DASHBOARD_PORT}} --redis-url redis://{{.REDIS_DEV_HOST}}:6379/0 dir: ../.. silent: true start-dev: - deps: [start-backend, start-worker] - desc: Start the backend and the worker + deps: [start-backend, start-worker, start-rq-dashboard] + desc: Start the backend, the worker and the RQ dashboard cmds: - - echo "Starting the backend and the worker" + - echo "Starting the backend, worker and RQ dashboard (http://localhost:{{.RQ_DASHBOARD_PORT}})" silent: true init-env: @@ -228,10 +256,3 @@ tasks: silent: true status: - test -f docker-compose.yml - - monitor-queues: - desc: Show the queues of the docker-compose API - cmds: - - uv run --package ecoindex_api celery --app=ecoindex.worker.tasks events - dir: ../.. - silent: true diff --git a/projects/ecoindex_api/docker-compose.yml.template b/projects/ecoindex_api/docker-compose.yml.template index 157dc0b..4300a37 100644 --- a/projects/ecoindex_api/docker-compose.yml.template +++ b/projects/ecoindex_api/docker-compose.yml.template @@ -46,6 +46,7 @@ services: DATABASE_URL: mysql+aiomysql://${DB_USER:-ecoindex}:${DB_PASSWORD:-ecoindex}@${DB_HOST:-db}/${DB_NAME:-ecoindex}?charset=utf8mb4 DEBUG: ${DEBUG:-0} REDIS_CACHE_HOST: ${REDIS_CACHE_HOST:-redis} + RQ_WORKERS: ${RQ_WORKERS:-3} TZ: ${TZ:-Europe/Paris} ENABLE_SCREENSHOT: ${ENABLE_SCREENSHOT:-0} depends_on: @@ -63,14 +64,6 @@ services: volumes: - redis:/data - flower: - image: mher/flower - ports: - - "${FLOWER_PORT:-5555}:5555" - environment: - CELERY_BROKER_URL: redis://${REDIS_CACHE_HOST:-redis}/0 - FLOWER_BASIC_AUTH: ${FLOWER_BASIC_AUTH:-ecoindex:ecoindex} - volumes: db: redis: diff --git a/projects/ecoindex_api/docker/worker/dockerfile b/projects/ecoindex_api/docker/worker/dockerfile index e8308fb..f961341 100644 --- a/projects/ecoindex_api/docker/worker/dockerfile +++ b/projects/ecoindex_api/docker/worker/dockerfile @@ -29,6 +29,7 @@ RUN playwright install chromium --with-deps RUN rm -rf $wheel requirements.txt /tmp/dist /var/lib/{apt,dpkg,cache,log}/ COPY projects/ecoindex_api/docker/worker/entrypoint.sh /usr/bin/entrypoint -RUN chmod +x /usr/bin/entrypoint +COPY projects/ecoindex_api/scripts/start_rq_workers.sh /usr/bin/start_rq_workers.sh +RUN chmod +x /usr/bin/entrypoint /usr/bin/start_rq_workers.sh ENTRYPOINT [ "/usr/bin/entrypoint" ] diff --git a/projects/ecoindex_api/docker/worker/entrypoint.sh b/projects/ecoindex_api/docker/worker/entrypoint.sh index 7e4080f..55e713b 100644 --- a/projects/ecoindex_api/docker/worker/entrypoint.sh +++ b/projects/ecoindex_api/docker/worker/entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/sh -celery -A ecoindex.worker.tasks worker --queues=ecoindex,ecoindex_batch \ No newline at end of file +exec /usr/bin/start_rq_workers.sh diff --git a/projects/ecoindex_api/openapi.json b/projects/ecoindex_api/openapi.json index 99ab3f2..f02165a 100644 --- a/projects/ecoindex_api/openapi.json +++ b/projects/ecoindex_api/openapi.json @@ -25,7 +25,7 @@ "type": "null" } ], - "default": "5.4.3", + "default": "5.10.0", "description": "Is the version of the ecoindex used to compute the score", "title": "Ecoindex version" }, @@ -155,6 +155,19 @@ "title": "Page size", "type": "number" }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "ecoindex.fr", + "description": "Source of the analysis", + "title": "Source of the analysis" + }, "url": { "description": "Url of the analysed page", "examples": [ @@ -218,6 +231,32 @@ "title": "BadgeTheme", "type": "string" }, + "Body_Add_new_ecoindex_analysis_task_to_the_waiting_queue_v1_tasks_ecoindexes__post": { + "properties": { + "custom_headers": { + "additionalProperties": { + "type": "string" + }, + "default": {}, + "description": "Custom headers to add to the request", + "title": "Custom Headers", + "type": "object" + }, + "web_page": { + "allOf": [ + { + "$ref": "#/components/schemas/WebPage" + } + ], + "title": "Web page to analyze defined by its url and its screen resolution" + } + }, + "required": [ + "web_page" + ], + "title": "Body_Add_new_ecoindex_analysis_task_to_the_waiting_queue_v1_tasks_ecoindexes__post", + "type": "object" + }, "Ecoindex": { "properties": { "ecoindex_version": { @@ -229,7 +268,7 @@ "type": "null" } ], - "default": "5.4.3", + "default": "5.10.0", "description": "Is the version of the ecoindex used to compute the score", "title": "Ecoindex version" }, @@ -335,7 +374,8 @@ "C", "D", "E", - "F" + "F", + "G" ], "title": "Grade", "type": "string" @@ -443,38 +483,7 @@ "title": "Host", "type": "object" }, - "PageApiEcoindexes-Input": { - "properties": { - "items": { - "items": { - "$ref": "#/components/schemas/ApiEcoindex" - }, - "title": "Items", - "type": "array" - }, - "page": { - "title": "Page", - "type": "integer" - }, - "size": { - "title": "Size", - "type": "integer" - }, - "total": { - "title": "Total", - "type": "integer" - } - }, - "required": [ - "items", - "total", - "page", - "size" - ], - "title": "PageApiEcoindexes", - "type": "object" - }, - "PageApiEcoindexes-Output": { + "PageApiEcoindexes": { "properties": { "items": { "items": { @@ -536,12 +545,12 @@ "title": "PageHosts", "type": "object" }, - "QueueTaskApi-Input": { + "QueueTaskApi": { "properties": { "ecoindex_result": { "anyOf": [ { - "$ref": "#/components/schemas/QueueTaskResult-Input" + "$ref": "#/components/schemas/QueueTaskResult" }, { "type": "null" @@ -553,6 +562,18 @@ "title": "Identifier of the current. This identifier will become the identifier of the analysis", "type": "string" }, + "queue_position": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "description": "Null when the task is running or already completed.", + "title": "Position of the task in the waiting queue (0 = next to run)" + }, "status": { "title": "Status of the current task. Can be PENDING, FAILURE, SUCCESS", "type": "string" @@ -565,6 +586,11 @@ } ], "title": "Detail of the error encountered by the task in case of Failure" + }, + "tasks_in_progress": { + "default": 0, + "title": "Number of tasks currently being processed on this queue", + "type": "integer" } }, "required": [ @@ -574,19 +600,8 @@ "title": "QueueTaskApi", "type": "object" }, - "QueueTaskApi-Output": { + "QueueTaskApiBatch": { "properties": { - "ecoindex_result": { - "anyOf": [ - { - "$ref": "#/components/schemas/QueueTaskResult-Output" - }, - { - "type": "null" - } - ], - "title": "Result of the Ecoindex analysis" - }, "id": { "title": "Identifier of the current. This identifier will become the identifier of the analysis", "type": "string" @@ -609,7 +624,7 @@ "id", "status" ], - "title": "QueueTaskApi", + "title": "QueueTaskApiBatch", "type": "object" }, "QueueTaskError": { @@ -663,47 +678,7 @@ "title": "QueueTaskError", "type": "object" }, - "QueueTaskResult-Input": { - "properties": { - "detail": { - "anyOf": [ - { - "$ref": "#/components/schemas/Result" - }, - { - "type": "null" - } - ], - "title": "Result of the ecoindex analysis once it was successfuly completed" - }, - "error": { - "anyOf": [ - { - "$ref": "#/components/schemas/QueueTaskError" - }, - { - "type": "null" - } - ], - "title": "Detail of the ecoindex error if it is not successful" - }, - "status": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "While the task is pending or the analysis is running, it is null. But once the analysis is complete, it should return SUCCESS or FAILURE.", - "title": "Status of the ecoindex analysis." - } - }, - "title": "QueueTaskResult", - "type": "object" - }, - "QueueTaskResult-Output": { + "QueueTaskResult": { "properties": { "detail": { "anyOf": [ @@ -767,7 +742,7 @@ "type": "null" } ], - "default": "5.4.3", + "default": "5.10.0", "description": "Is the version of the ecoindex used to compute the score", "title": "Ecoindex version" }, @@ -994,7 +969,7 @@ "info": { "description": "Ecoindex API enables you to perform ecoindex analysis of given web pages", "title": "Ecoindex API", - "version": "3.1.0" + "version": "3.12.0" }, "openapi": "3.1.0", "paths": { @@ -1094,25 +1069,20 @@ ] } }, - "/v1/tasks/ecoindexes": { + "/v1/tasks/ecoindexes/": { "post": { "description": "This submits a ecoindex analysis task to the engine", - "operationId": "Add_new_ecoindex_analysis_task_to_the_waiting_queue_v1_tasks_ecoindexes_post", + "operationId": "Add_new_ecoindex_analysis_task_to_the_waiting_queue_v1_tasks_ecoindexes__post", "requestBody": { "content": { "application/json": { - "example": { - "height": 1080, - "url": "https://www.ecoindex.fr/", - "width": 1920 - }, "schema": { "allOf": [ { - "$ref": "#/components/schemas/WebPage" + "$ref": "#/components/schemas/Body_Add_new_ecoindex_analysis_task_to_the_waiting_queue_v1_tasks_ecoindexes__post" } ], - "title": "Web page to analyze defined by its url and its screen resolution" + "title": "Body" } } }, @@ -1123,7 +1093,7 @@ "content": { "application/json": { "schema": { - "title": "Response 201 Add New Ecoindex Analysis Task To The Waiting Queue V1 Tasks Ecoindexes Post", + "title": "Response 201 Add New Ecoindex Analysis Task To The Waiting Queue V1 Tasks Ecoindexes Post", "type": "string" } } @@ -1134,7 +1104,7 @@ "content": { "application/json": { "schema": { - "title": "Response 403 Add New Ecoindex Analysis Task To The Waiting Queue V1 Tasks Ecoindexes Post", + "title": "Response 403 Add New Ecoindex Analysis Task To The Waiting Queue V1 Tasks Ecoindexes Post", "type": "string" } } @@ -1184,6 +1154,16 @@ } }, "description": "You have reached the daily limit" + }, + "521": { + "content": { + "application/json": { + "example": { + "detail": "The URL http://localhost is unreachable. Are you really sure of this url? \ud83e\udd14" + } + } + }, + "description": "Host unreachable" } }, "summary": "Add New Ecoindex Analysis Task To The Waiting Queue", @@ -1192,6 +1172,143 @@ ] } }, + "/v1/tasks/ecoindexes/batch": { + "post": { + "description": "This save ecoindex analysis from external source in batch mode. Limited to 100 entries at a time", + "operationId": "Save_ecoindex_analysis_from_external_source_in_batch_mode_v1_tasks_ecoindexes_batch_post", + "parameters": [ + { + "in": "header", + "name": "X-Api-Key", + "required": true, + "schema": { + "title": "X-Api-Key", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "example": [], + "schema": { + "items": { + "$ref": "#/components/schemas/ApiEcoindex" + }, + "maxItems": 100, + "minItems": 1, + "title": "List of ecoindex analysis results to save", + "type": "array" + } + } + }, + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "title": "Response 201 Save Ecoindex Analysis From External Source In Batch Mode V1 Tasks Ecoindexes Batch Post", + "type": "string" + } + } + }, + "description": "Identifier of the task that has been created in queue" + }, + "403": { + "content": { + "application/json": { + "schema": { + "title": "Response 403 Save Ecoindex Analysis From External Source In Batch Mode V1 Tasks Ecoindexes Batch Post", + "type": "string" + } + } + }, + "description": "Forbidden" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Save Ecoindex Analysis From External Source In Batch Mode", + "tags": [ + "Tasks" + ] + } + }, + "/v1/tasks/ecoindexes/batch/{id}": { + "get": { + "description": "This returns an ecoindex batch task result given by its unique identifier", + "operationId": "Get_ecoindex_analysis_batch_task_by_id_v1_tasks_ecoindexes_batch__id__get", + "parameters": [ + { + "description": "Unique identifier of the ecoindex analysis", + "in": "path", + "name": "id", + "required": true, + "schema": { + "description": "Unique identifier of the ecoindex analysis", + "format": "uuid", + "title": "Id", + "type": "string" + } + }, + { + "in": "header", + "name": "X-Api-Key", + "required": true, + "schema": { + "title": "X-Api-Key", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueueTaskApiBatch" + } + } + }, + "description": "Get one ecoindex batch task result by its id" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + }, + "425": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/QueueTaskApiBatch" + } + } + }, + "description": "Too Early" + } + }, + "summary": "Get Ecoindex Analysis Batch Task By Id", + "tags": [ + "Tasks" + ] + } + }, "/v1/tasks/ecoindexes/{id}": { "delete": { "description": "This aborts one ecoindex task by its id if it is still waiting", @@ -1257,7 +1374,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/QueueTaskApi-Input" + "$ref": "#/components/schemas/QueueTaskApi" } } }, @@ -1277,7 +1394,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/QueueTaskApi-Input" + "$ref": "#/components/schemas/QueueTaskApi" } } }, @@ -1417,7 +1534,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PageApiEcoindexes-Output" + "$ref": "#/components/schemas/PageApiEcoindexes" } } }, @@ -1427,7 +1544,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PageApiEcoindexes-Input" + "$ref": "#/components/schemas/PageApiEcoindexes" } } }, @@ -1437,7 +1554,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PageApiEcoindexes-Input" + "$ref": "#/components/schemas/PageApiEcoindexes" } } }, @@ -1531,7 +1648,6 @@ }, "summary": "Get Latest Results", "tags": [ - "Ecoindex", "BFF" ] } @@ -1631,7 +1747,6 @@ }, "summary": "Get Badge", "tags": [ - "Ecoindex", "BFF" ] } @@ -1705,7 +1820,6 @@ }, "summary": "Get Latest Results Redirect", "tags": [ - "Ecoindex", "BFF" ] } @@ -1880,34 +1994,34 @@ "name": "q", "required": false, "schema": { + "deprecated": true, "description": "Filter by partial host name (replaced by `host`)", "title": "Q", "type": "string" } }, { - "description": "Start date of the filter elements (example: 2020-01-01)", + "description": "Host name you want to filter (can be partial)", "in": "query", - "name": "date_from", + "name": "host", "required": false, "schema": { "anyOf": [ { - "format": "date", "type": "string" }, { "type": "null" } ], - "description": "Start date of the filter elements (example: 2020-01-01)", - "title": "Date From" + "description": "Host name you want to filter (can be partial)", + "title": "Host" } }, { - "description": "End date of the filter elements (example: 2020-01-01)", + "description": "Start date of the filter elements (example: 2020-01-01)", "in": "query", - "name": "date_to", + "name": "date_from", "required": false, "schema": { "anyOf": [ @@ -1919,26 +2033,27 @@ "type": "null" } ], - "description": "End date of the filter elements (example: 2020-01-01)", - "title": "Date To" + "description": "Start date of the filter elements (example: 2020-01-01)", + "title": "Date From" } }, { - "description": "Host name you want to filter (can be partial)", + "description": "End date of the filter elements (example: 2020-01-01)", "in": "query", - "name": "host", + "name": "date_to", "required": false, "schema": { "anyOf": [ { + "format": "date", "type": "string" }, { "type": "null" } ], - "description": "Host name you want to filter (can be partial)", - "title": "Host" + "description": "End date of the filter elements (example: 2020-01-01)", + "title": "Date To" } }, { diff --git a/projects/ecoindex_api/pyproject.toml b/projects/ecoindex_api/pyproject.toml index f980b32..769c381 100644 --- a/projects/ecoindex_api/pyproject.toml +++ b/projects/ecoindex_api/pyproject.toml @@ -9,7 +9,7 @@ authors = [{ name = "Vincent Vatelot", email = "vincent.vatelot@ik.me" }] dependencies = [ "aiofile>=3.8.8", "alembic>=1.12.1", - "celery>=5.3.4", + "rq>=2.1.0", "cryptography>=44.0.2", "fastapi>=0.109.1", "playwright>=1.39.0", @@ -46,6 +46,7 @@ worker = [ ] dev = [ "aiosqlite>=0.19.0", + "rq-dashboard @ git+https://github.com/Parallels/rq-dashboard.git", "typing-extensions>=4.8.0", "watchdog>=6.0.0", ] diff --git a/projects/ecoindex_api/scripts/start_rq_workers.sh b/projects/ecoindex_api/scripts/start_rq_workers.sh new file mode 100755 index 0000000..49c89ba --- /dev/null +++ b/projects/ecoindex_api/scripts/start_rq_workers.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -e + +RQ_WORKERS="${RQ_WORKERS:-3}" +REDIS_URL="${RQ_REDIS_URL:-redis://${REDIS_CACHE_HOST:-localhost}:6379/0}" + +start_worker() { + if [ "${USE_UV:-0}" = "1" ]; then + uv run --package ecoindex_api --group worker rq worker "$@" + else + rq worker "$@" + fi +} + +i=1 +while [ "$i" -le "$RQ_WORKERS" ]; do + start_worker ecoindex ecoindex_batch \ + --url "$REDIS_URL" \ + --name "ecoindex-worker-$i" & + i=$((i + 1)) +done + +wait diff --git a/projects/ecoindex_api/scripts/wait_redis.py b/projects/ecoindex_api/scripts/wait_redis.py new file mode 100644 index 0000000..e10885a --- /dev/null +++ b/projects/ecoindex_api/scripts/wait_redis.py @@ -0,0 +1,70 @@ +"""Dev helper: resolve a reachable Redis host for local development.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time + +from redis import Redis +from redis.exceptions import ConnectionError + +CONTAINER = "ecoindex-dev-redis" +DEFAULT_GATEWAY_HOST = "172.17.0.1" + + +def candidate_hosts() -> list[str]: + if os.path.exists("/.dockerenv"): + hosts = [DEFAULT_GATEWAY_HOST, "localhost", "127.0.0.1"] + else: + hosts = ["localhost", "127.0.0.1", DEFAULT_GATEWAY_HOST] + try: + data = json.loads( + subprocess.check_output( + ["docker", "inspect", CONTAINER], + stderr=subprocess.DEVNULL, + ) + )[0] + networks = data.get("NetworkSettings", {}).get("Networks", {}) + if networks: + ip = next(iter(networks.values()))["IPAddress"] + hosts.append(ip) + except ( + subprocess.CalledProcessError, + StopIteration, + KeyError, + IndexError, + json.JSONDecodeError, + ): + pass + return list(dict.fromkeys(hosts)) + + +def ping(host: str) -> bool: + try: + return Redis(host=host, socket_connect_timeout=1).ping() + except ConnectionError: + return False + + +def resolve_host() -> str: + for host in candidate_hosts(): + if ping(host): + return host + raise RuntimeError("Redis is not reachable") + + +def main() -> None: + for _ in range(50): + try: + print(resolve_host()) + return + except RuntimeError: + time.sleep(0.2) + raise SystemExit("Redis is not ready") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index fa7b3cd..95039eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ cli = [ api = [ "aiosqlite>=0.19.0", "alembic>=1.12.1", - "celery>=5.3.4", + "rq>=2.1.0", "sentry-sdk>=2.8.0", "sqlmodel>=0.0.14", "ua-generator>=2.0.3", diff --git a/uv.lock b/uv.lock index 3919492..0ae1a7b 100644 --- a/uv.lock +++ b/uv.lock @@ -25,7 +25,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "caio", marker = "python_full_version < '3.11'" }, + { name = "caio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/e2/d7cb819de8df6b5c1968a2756c3cb4122d4fa2b8fc768b53b7c9e5edb646/aiofile-3.9.0.tar.gz", hash = "sha256:e5ad718bb148b265b6df1b3752c4d1d83024b93da9bd599df74b9d9ffcf7919b", size = 17943, upload-time = "2024-10-08T10:39:35.846Z" } wheels = [ @@ -42,7 +42,7 @@ resolution-markers = [ "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "caio", marker = "python_full_version >= '3.11'" }, + { name = "caio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/41/2fea7e193e061ce54eacc3b7bc0e6a99e4fcff43c78cf0a76dd781ed8334/aiofile-3.11.1.tar.gz", hash = "sha256:1f91912c6643d2a4e49ca4ae3514f0bf3867ce948a36d99a6411b8f4755f4cf9", size = 19342, upload-time = "2026-05-16T08:18:33.538Z" } wheels = [ @@ -73,18 +73,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893, upload-time = "2026-02-10T16:00:49.997Z" }, ] -[[package]] -name = "amqp" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/79/fc/ec94a357dfc6683d8c86f8b4cfa5416a4c36b28052ec8260c77aca96a443/amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432", size = 129013, upload-time = "2024-11-12T19:55:44.051Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944, upload-time = "2024-11-12T19:55:41.782Z" }, -] - [[package]] name = "annotated-doc" version = "0.0.4" @@ -117,6 +105,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, ] +[[package]] +name = "arrow" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/33/032cdc44182491aa708d06a68b62434140d8c50820a087fac7af37703357/arrow-1.4.0.tar.gz", hash = "sha256:ed0cc050e98001b8779e84d461b0098c4ac597e88704a655582b21d116e526d7", size = 152931, upload-time = "2025-10-18T17:46:46.761Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/c9/d7977eaacb9df673210491da99e6a247e93df98c715fc43fd136ce1d3d33/arrow-1.4.0-py3-none-any.whl", hash = "sha256:749f0769958ebdc79c173ff0b0670d59051a535fa26e8eba02953dc19eb43205", size = 68797, upload-time = "2025-10-18T17:46:45.663Z" }, +] + [[package]] name = "ast-serialize" version = "0.5.0" @@ -177,15 +178,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/59/76ab57e3fe74484f48a53f8e337171b4a2349e506eabe136d7e01d059086/backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5", size = 12313, upload-time = "2025-07-02T02:27:14.263Z" }, ] -[[package]] -name = "billiard" -version = "4.2.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/58/23/b12ac0bcdfb7360d664f40a00b1bda139cbbbced012c34e375506dbd0143/billiard-4.2.4.tar.gz", hash = "sha256:55f542c371209e03cd5862299b74e52e4fbcba8250ba611ad94276b369b6a85f", size = 156537, upload-time = "2025-11-30T13:28:48.52Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/87/8bab77b323f16d67be364031220069f79159117dd5e43eeb4be2fef1ac9b/billiard-4.2.4-py3-none-any.whl", hash = "sha256:525b42bdec68d2b983347ac312f892db930858495db601b5836ac24e6477cde5", size = 87070, upload-time = "2025-11-30T13:28:47.016Z" }, -] - [[package]] name = "black" version = "26.5.1" @@ -220,6 +212,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/51/f975cae76d44274cc2868dc9040ac5d58d464784610234455b4e7b19c6ef/black-26.5.1-py3-none-any.whl", hash = "sha256:4ed7f7da04046d2e488437170797d3b4a4ad83906683bcb7dfc68b673bbce5e2", size = 213693, upload-time = "2026-05-18T16:53:33.964Z" }, ] +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, +] + [[package]] name = "cached-property" version = "2.0.1" @@ -250,27 +251,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/93/1f76c8d1bafe3b0614e06b2195784a3765bbf7b0a067661af9e2dd47fc33/caio-0.9.25-py3-none-any.whl", hash = "sha256:06c0bb02d6b929119b1cfbe1ca403c768b2013a369e2db46bfa2a5761cf82e40", size = 19087, upload-time = "2025-12-26T15:22:00.221Z" }, ] -[[package]] -name = "celery" -version = "5.6.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "billiard" }, - { name = "click" }, - { name = "click-didyoumean" }, - { name = "click-plugins" }, - { name = "click-repl" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "kombu" }, - { name = "python-dateutil" }, - { name = "tzlocal" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e8/b4/a1233943ab5c8ea05fb877a88a0a0622bf47444b99e4991a8045ac37ea1d/celery-5.6.3.tar.gz", hash = "sha256:177006bd2054b882e9f01be59abd8529e88879ef50d7918a7050c5a9f4e12912", size = 1742243, upload-time = "2026-03-26T12:14:51.76Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/c9/6eccdda96e098f7ae843162db2d3c149c6931a24fda69fe4ab84d0027eb5/celery-5.6.3-py3-none-any.whl", hash = "sha256:0808f42f80909c4d5833202360ffafb2a4f83f4d8e23e1285d926610e9a7afa6", size = 451235, upload-time = "2026-03-26T12:14:49.491Z" }, -] - [[package]] name = "certifi" version = "2026.5.20" @@ -397,43 +377,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, ] -[[package]] -name = "click-didyoumean" -version = "0.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/30/ce/217289b77c590ea1e7c24242d9ddd6e249e52c795ff10fac2c50062c48cb/click_didyoumean-0.3.1.tar.gz", hash = "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463", size = 3089, upload-time = "2024-03-24T08:22:07.499Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/5b/974430b5ffdb7a4f1941d13d83c64a0395114503cc357c6b9ae4ce5047ed/click_didyoumean-0.3.1-py3-none-any.whl", hash = "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c", size = 3631, upload-time = "2024-03-24T08:22:06.356Z" }, -] - -[[package]] -name = "click-plugins" -version = "1.1.1.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/9a/2abecb28ae875e39c8cad711eb1186d8d14eab564705325e77e4e6ab9ae5/click_plugins-1.1.1.2-py2.py3-none-any.whl", hash = "sha256:008d65743833ffc1f5417bf0e78e8d2c23aab04d9745ba817bd3e71b0feb6aa6", size = 11051, upload-time = "2025-06-25T00:47:36.731Z" }, -] - -[[package]] -name = "click-repl" -version = "0.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "prompt-toolkit" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/cb/a2/57f4ac79838cfae6912f997b4d1a64a858fb0c86d7fcaae6f7b58d267fca/click-repl-0.3.0.tar.gz", hash = "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9", size = 10449, upload-time = "2023-06-15T12:43:51.141Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/40/9d857001228658f0d59e97ebd4c346fe73e138c6de1bce61dc568a57c7f8/click_repl-0.3.0-py3-none-any.whl", hash = "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812", size = 10289, upload-time = "2023-06-15T12:43:48.626Z" }, -] - [[package]] name = "click-spinner" version = "0.1.10" @@ -469,7 +412,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } wheels = [ @@ -521,7 +464,7 @@ resolution-markers = [ "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -612,6 +555,18 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11'" }, ] +[[package]] +name = "croniter" +version = "6.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/03/35/96ad0a71eb0b27ab4476a7ed23facd0713d82da9c911edc8af7f34a62d6a/croniter-6.2.3.tar.gz", hash = "sha256:fb129986ef7e2c44e3f4c9f503da83ad914d2afa48f40a43ee3dca4b5c41d476", size = 166174, upload-time = "2026-07-02T14:34:22.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/dd/6466498a8b69754cffbd7237ce4c66446ca5ffcf53fb397d437666f056d2/croniter-6.2.3-py3-none-any.whl", hash = "sha256:137a97001b4d52fb71c10b750e303db79e6e42d40fff8ff77126102176c9f786", size = 46446, upload-time = "2026-07-02T14:34:20.889Z" }, +] + [[package]] name = "cryptography" version = "48.0.0" @@ -696,13 +651,12 @@ wheels = [ [[package]] name = "ecoindex-api" -version = "3.11.1" +version = "3.12.0" source = { editable = "projects/ecoindex_api" } dependencies = [ { name = "aiofile", version = "3.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "aiofile", version = "3.11.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "alembic" }, - { name = "celery" }, { name = "cryptography" }, { name = "fastapi" }, { name = "playwright" }, @@ -712,6 +666,7 @@ dependencies = [ { name = "pyyaml" }, { name = "redis" }, { name = "requests" }, + { name = "rq" }, { name = "sentry-sdk" }, { name = "setuptools" }, { name = "sqlmodel" }, @@ -729,6 +684,7 @@ backend = [ ] dev = [ { name = "aiosqlite" }, + { name = "rq-dashboard" }, { name = "typing-extensions" }, { name = "watchdog" }, ] @@ -742,7 +698,6 @@ worker = [ requires-dist = [ { name = "aiofile", specifier = ">=3.8.8" }, { name = "alembic", specifier = ">=1.12.1" }, - { name = "celery", specifier = ">=5.3.4" }, { name = "cryptography", specifier = ">=44.0.2" }, { name = "fastapi", specifier = ">=0.109.1" }, { name = "pillow", marker = "extra == 'webp'", specifier = ">=12.2.0" }, @@ -753,6 +708,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.1" }, { name = "redis", specifier = ">=5.0.1" }, { name = "requests", specifier = ">=2.32.4" }, + { name = "rq", specifier = ">=2.1.0" }, { name = "sentry-sdk", specifier = ">=2.8.0" }, { name = "setuptools", specifier = ">=78.1.1" }, { name = "sqlmodel", specifier = ">=0.0.14" }, @@ -764,6 +720,7 @@ provides-extras = ["webp"] backend = [{ name = "uvicorn", specifier = ">=0.23.2" }] dev = [ { name = "aiosqlite", specifier = ">=0.19.0" }, + { name = "rq-dashboard", git = "https://github.com/Parallels/rq-dashboard.git" }, { name = "typing-extensions", specifier = ">=4.8.0" }, { name = "watchdog", specifier = ">=6.0.0" }, ] @@ -775,7 +732,7 @@ worker = [ [[package]] name = "ecoindex-cli" -version = "2.30.0" +version = "2.30.1" source = { editable = "projects/ecoindex_cli" } dependencies = [ { name = "aiofile", version = "3.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -820,7 +777,7 @@ requires-dist = [ [[package]] name = "ecoindex-compute" -version = "5.9.0" +version = "5.10.0" source = { editable = "projects/ecoindex_compute" } dependencies = [ { name = "aiofile", version = "3.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, @@ -859,7 +816,7 @@ dependencies = [ api = [ { name = "aiosqlite" }, { name = "alembic" }, - { name = "celery" }, + { name = "rq" }, { name = "sentry-sdk" }, { name = "sqlmodel" }, { name = "ua-generator" }, @@ -921,7 +878,7 @@ requires-dist = [ api = [ { name = "aiosqlite", specifier = ">=0.19.0" }, { name = "alembic", specifier = ">=1.12.1" }, - { name = "celery", specifier = ">=5.3.4" }, + { name = "rq", specifier = ">=2.1.0" }, { name = "sentry-sdk", specifier = ">=2.8.0" }, { name = "sqlmodel", specifier = ">=0.0.14" }, { name = "ua-generator", specifier = ">=2.0.3" }, @@ -963,7 +920,7 @@ scraper-webp = [{ name = "pillow", specifier = ">=12.2.0" }] [[package]] name = "ecoindex-scraper" -version = "3.16.0" +version = "3.17.0" source = { editable = "projects/ecoindex_scraper" } dependencies = [ { name = "playwright" }, @@ -1011,7 +968,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1042,6 +999,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, ] +[[package]] +name = "flask" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blinker" }, + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "markupsafe" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/00/35d85dcce6c57fdc871f3867d465d780f302a175ea360f62533f12b27e2b/flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb", size = 759004, upload-time = "2026-02-19T05:00:57.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, +] + [[package]] name = "fonttools" version = "4.63.0" @@ -1247,6 +1221,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/71/d9cd0e4c6a4aace991009fc47362ce9251be0fbcf2b6c533f918b31854d5/itemloaders-1.4.0-py3-none-any.whl", hash = "sha256:202b6f855299b4cadfdf78bb93a6cf977899e3c40c4c54524e120a444e65b5ac", size = 12188, upload-time = "2026-01-29T12:50:36.148Z" }, ] +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -1333,21 +1316,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/dd/8050c947d435c8d4bc94e3252f4d8bb8a76cfb424f043a8680be637a57f1/kiwisolver-1.5.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:59cd8683f575d96df5bb48f6add94afc055012c29e28124fcae2b63661b9efb1", size = 73558, upload-time = "2026-03-09T13:15:52.112Z" }, ] -[[package]] -name = "kombu" -version = "5.6.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "amqp" }, - { name = "packaging" }, - { name = "tzdata" }, - { name = "vine" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b6/a5/607e533ed6c83ae1a696969b8e1c137dfebd5759a2e9682e26ff1b97740b/kombu-5.6.2.tar.gz", hash = "sha256:8060497058066c6f5aed7c26d7cd0d3b574990b09de842a8c5aaed0b92cc5a55", size = 472594, upload-time = "2025-12-29T20:30:07.779Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, -] - [[package]] name = "librt" version = "0.11.0" @@ -1743,10 +1711,10 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "python-dateutil", marker = "python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -1783,9 +1751,9 @@ resolution-markers = [ "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -1945,18 +1913,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c2/2c7a42c08954622a061b51c35f90c97d9b87f505438401aeda5c38a594a1/polylith_cli-1.47.2-py3-none-any.whl", hash = "sha256:d08f9dc1e8877f5f22eb937de368a9e869aea40fd6b6fed636af36375c1f9d4e", size = 101222, upload-time = "2026-05-06T20:42:57.128Z" }, ] -[[package]] -name = "prompt-toolkit" -version = "3.0.52" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "wcwidth" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, -] - [[package]] name = "protego" version = "0.6.0" @@ -2308,6 +2264,18 @@ hiredis = [ { name = "hiredis" }, ] +[[package]] +name = "redis-sentinel-url" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "redis" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/e3/68de4c8aacbd9952667d7d0f5af55b873553a09a6227ac5b7f7c622610c9/Redis-Sentinel-Url-1.0.1.tar.gz", hash = "sha256:ec1854ab9379a28789423c3cd4082739fb69c7b4b11bb50ae7858697c131b13e", size = 7599, upload-time = "2017-04-05T07:47:55.456Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/35/20f997f367c87ef1e6ebf418af7c2450cb5da860d066d7229226031534ad/Redis_Sentinel_Url-1.0.1-py2.py3-none-any.whl", hash = "sha256:c6991e2000c5c7a5e2b95eb2d62fd5b0a6b02a59554caf0f9f79d18e152d9663", size = 4685, upload-time = "2017-04-05T07:47:53.545Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -2348,6 +2316,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/76/6d163cfac87b632216f71879e6b2cf17163f773ff59c00b5ff4900a80fa3/rich-14.3.4-py3-none-any.whl", hash = "sha256:07e7adb4690f68864777b1450859253bed81a99a31ac321ac1817b2313558952", size = 310480, upload-time = "2026-04-11T02:57:47.484Z" }, ] +[[package]] +name = "rq" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "croniter" }, + { name = "redis" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/a8/7bf65cda593feb5888214a4d1e64aae6fc5eb0bbbb74df922c916099a3e5/rq-2.10.0.tar.gz", hash = "sha256:2d8c533dd27500fedabec06295f18db595966e4f22744e6988fe31155b8f7a21", size = 754610, upload-time = "2026-06-20T03:11:45.919Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/d3/f8c59750a1c98f99d225954eb2c3dd89f2d66b666c448b4b0dff79ab29ed/rq-2.10.0-py3-none-any.whl", hash = "sha256:1f072e0ff79771f3d3a810170df4c4836dca9ce9f31330a9fc25e25277620ec8", size = 124665, upload-time = "2026-06-20T03:11:47.524Z" }, +] + +[[package]] +name = "rq-dashboard" +version = "0.9.0" +source = { git = "https://github.com/Parallels/rq-dashboard.git#3735f1ebb6a936ff5ad1ef6b3a3bfae58bc03029" } +dependencies = [ + { name = "arrow" }, + { name = "flask" }, + { name = "redis" }, + { name = "redis-sentinel-url" }, + { name = "rq" }, +] + [[package]] name = "ruff" version = "0.15.15" @@ -2656,18 +2650,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, ] -[[package]] -name = "tzlocal" -version = "5.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tzdata", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, -] - [[package]] name = "ua-generator" version = "2.1.1" @@ -2700,15 +2682,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/be/72532be3da7acc5fdfbccdb95215cd04f995a0886532a5b423f929cda4cc/uvicorn-0.48.0-py3-none-any.whl", hash = "sha256:48097851328b87ec36117d3d575234519eb58c2b22d79666e9bbc6c49a761dad", size = 71410, upload-time = "2026-05-24T12:08:40.258Z" }, ] -[[package]] -name = "vine" -version = "5.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bd/e4/d07b5f29d283596b9727dd5275ccbceb63c44a1a82aa9e4bfd20426762ac/vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0", size = 48980, upload-time = "2023-11-05T08:46:53.857Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/ff/7c0c86c43b3cbb927e0ccc0255cb4057ceba4799cd44ae95174ce8e8b5b2/vine-5.1.0-py3-none-any.whl", hash = "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc", size = 9636, upload-time = "2023-11-05T08:46:51.205Z" }, -] - [[package]] name = "w3lib" version = "2.4.1" @@ -2748,12 +2721,15 @@ wheels = [ ] [[package]] -name = "wcwidth" -version = "0.7.0" +name = "werkzeug" +version = "3.1.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/ee/afaf0f85a9a18fe47a67f1e4422ed6cf1fe642f0ae0a2f81166231303c52/wcwidth-0.7.0.tar.gz", hash = "sha256:90e3a7ea092341c44b99562e75d09e4d5160fe7a3974c6fb842a101a95e7eed0", size = 182132, upload-time = "2026-05-02T16:04:12.653Z" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/b2/381be8cfdee792dd117872481b6e378f85c957dd7c5bca38897b08f765fd/werkzeug-3.1.8.tar.gz", hash = "sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44", size = 875852, upload-time = "2026-04-02T18:49:14.268Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/52/e465037f5375f43533d1a80b6923955201596a99142ed524d77b571a1418/wcwidth-0.7.0-py3-none-any.whl", hash = "sha256:5d69154c429a82910e241c738cd0e2976fac8a2dd47a1a805f4afed1c0f136f2", size = 110825, upload-time = "2026-05-02T16:04:11.033Z" }, + { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, ] [[package]] From 0fbbd651f5c5e4c0cf7b91a52bb4bd892388d2fa Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Mon, 6 Jul 2026 15:01:08 +0000 Subject: [PATCH 2/5] fix(api): defer Redis resolution to runtime in dev tasks Avoid resolving the Redis host when loading the API Taskfile so CI tasks like project-check can run without a Redis instance. Co-authored-by: Cursor --- projects/ecoindex_api/Taskfile.yml | 32 +++++++++++++++++------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/projects/ecoindex_api/Taskfile.yml b/projects/ecoindex_api/Taskfile.yml index 76556d1..6fc1102 100644 --- a/projects/ecoindex_api/Taskfile.yml +++ b/projects/ecoindex_api/Taskfile.yml @@ -15,8 +15,6 @@ vars: RQ_DASHBOARD_PORT: "9181" RQ_WORKERS: sh: printf '%s' "${RQ_WORKERS:-3}" - REDIS_DEV_HOST: - sh: uv run python scripts/wait_redis.py tasks: update-openapi: @@ -183,24 +181,25 @@ tasks: start-worker: deps: [start-redis] - env: - REDIS_CACHE_HOST: "{{.REDIS_DEV_HOST}}" - PYTHONPATH: bases:components - RQ_WORKERS: "{{.RQ_WORKERS}}" - RQ_REDIS_URL: redis://{{.REDIS_DEV_HOST}}:6379/0 - USE_UV: "1" cmds: - - uv run --package ecoindex_api --group dev --group worker watchmedo auto-restart --directory=bases --directory=components --pattern=*.py --recursive -- bash projects/ecoindex_api/scripts/start_rq_workers.sh + - | + REDIS_DEV_HOST="$(uv run python scripts/wait_redis.py)" + export REDIS_CACHE_HOST="$REDIS_DEV_HOST" + export PYTHONPATH="bases:components" + export RQ_WORKERS="{{.RQ_WORKERS}}" + export RQ_REDIS_URL="redis://$REDIS_DEV_HOST:6379/0" + export USE_UV="1" + uv run --package ecoindex_api --group dev --group worker watchmedo auto-restart --directory=bases --directory=components --pattern=*.py --recursive -- bash projects/ecoindex_api/scripts/start_rq_workers.sh dir: ../.. silent: true start-backend: deps: [start-redis] - env: - REDIS_CACHE_HOST: "{{.REDIS_DEV_HOST}}" - PYTHONPATH: bases:components cmds: - - uv run --package ecoindex_api --group backend uvicorn ecoindex.backend.main:app --host 0.0.0.0 --port 8000 --reload --reload-dir bases --reload-dir components + - | + export REDIS_CACHE_HOST="$(uv run python scripts/wait_redis.py)" + export PYTHONPATH="bases:components" + uv run --package ecoindex_api --group backend uvicorn ecoindex.backend.main:app --host 0.0.0.0 --port 8000 --reload --reload-dir bases --reload-dir components dir: ../.. silent: true @@ -208,7 +207,12 @@ tasks: desc: Start the RQ dashboard for monitoring queues deps: [start-redis] cmds: - - uv run --package ecoindex_api --group dev rq-dashboard --bind 0.0.0.0 --port {{.RQ_DASHBOARD_PORT}} --redis-url redis://{{.REDIS_DEV_HOST}}:6379/0 + - | + REDIS_DEV_HOST="$(uv run python scripts/wait_redis.py)" + uv run --package ecoindex_api --group dev rq-dashboard \ + --bind 0.0.0.0 \ + --port {{.RQ_DASHBOARD_PORT}} \ + --redis-url "redis://$REDIS_DEV_HOST:6379/0" dir: ../.. silent: true From 7cc6e029ace2649c5232d4853a73de5b8613706e Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Tue, 7 Jul 2026 08:21:09 +0000 Subject: [PATCH 3/5] fix(api): resolve quality issues in RQ migration Add explicit enqueue settings typing for mypy and remove an unused import in the Redis helper so quality checks pass again. Co-authored-by: Cursor --- bases/ecoindex/backend/routers/tasks.py | 4 ++-- projects/ecoindex_api/scripts/wait_redis.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bases/ecoindex/backend/routers/tasks.py b/bases/ecoindex/backend/routers/tasks.py index 62e18c8..48454fc 100644 --- a/bases/ecoindex/backend/routers/tasks.py +++ b/bases/ecoindex/backend/routers/tasks.py @@ -80,9 +80,9 @@ def convert_url_to_punycode(url: str) -> str: return url -def _enqueue_settings(*, with_retry: bool = True) -> dict: +def _enqueue_settings(*, with_retry: bool = True) -> dict[str, object]: settings = Settings() - enqueue_settings = { + enqueue_settings: dict[str, object] = { "result_ttl": settings.RQ_RESULT_TTL, "failure_ttl": settings.RQ_FAILURE_TTL, "job_timeout": settings.RQ_JOB_TIMEOUT, diff --git a/projects/ecoindex_api/scripts/wait_redis.py b/projects/ecoindex_api/scripts/wait_redis.py index e10885a..19931ec 100644 --- a/projects/ecoindex_api/scripts/wait_redis.py +++ b/projects/ecoindex_api/scripts/wait_redis.py @@ -5,7 +5,6 @@ import json import os import subprocess -import sys import time from redis import Redis From 758edb98a1e2228be2178ca110fb2ae77ac1bc52 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Tue, 7 Jul 2026 12:59:16 +0000 Subject: [PATCH 4/5] chore(api): update RQ-dashboard dependency and improve devcontainer setup - Changed RQ-dashboard dependency from a Git URL to a PyPI source in multiple configuration files for consistency. - Enhanced devcontainer configuration by adding host mapping and forwarding ports for better development experience. - Updated Dockerfile to include docker-compose-plugin for improved container management. - Adjusted task scripts to reference the correct path for Redis wait script and added a new script for cleaning stale RQ workers. - Made database migration scripts more robust by checking for existing columns before adding or dropping them. --- .devcontainer/Dockerfile | 3 +- .devcontainer/devcontainer.json | 6 +- projects/ecoindex_api/Taskfile.yml | 6 +- .../alembic/versions/5afa2faea43f_.py | 13 ++-- .../e83263a5def4_add_index_id_and_host.py | 4 +- .../versions/fd9a1f5662c8_first_migration.py | 2 +- .../ecoindex_api/docker/worker/dockerfile | 1 + projects/ecoindex_api/pyproject.toml | 2 +- .../scripts/clean_stale_rq_workers.py | 71 +++++++++++++++++++ .../ecoindex_api/scripts/start_rq_workers.sh | 12 +++- tasks/DockerTaskfile.yml | 10 ++- uv.lock | 8 ++- 12 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 projects/ecoindex_api/scripts/clean_stale_rq_workers.py diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 17142f5..7784aff 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,13 +9,14 @@ RUN curl -fsSL https://taskfile.dev/install.sh | sh -s -- -d -b /usr/local/bin # The base image ships a Yarn apt source with an expired/missing GPG key, which breaks # apt-get for any devcontainer feature (e.g. docker-in-docker). Remove it before apt installs. # Use docker-ce-cli (not docker.io): the host daemon is bind-mounted and Docker 29+ requires API 1.44+. +# Install docker-compose-plugin so `docker compose` is available inside the devcontainer. RUN rm -f /etc/apt/sources.list.d/yarn*.list 2>/dev/null || true \ && install -m 0755 -d /etc/apt/keyrings \ && curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc \ && chmod a+r /etc/apt/keyrings/docker.asc \ && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" > /etc/apt/sources.list.d/docker.list \ && apt-get update \ - && apt-get install -y --no-install-recommends docker-ce-cli \ + && apt-get install -y --no-install-recommends docker-ce-cli docker-compose-plugin \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 16075a7..d569d06 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -9,10 +9,14 @@ "mounts": [ "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind" ], - "runArgs": ["--name=ecoindex-python-dev"], + "runArgs": [ + "--name=ecoindex-python-dev", + "--add-host=host.docker.internal:host-gateway" + ], "containerEnv": { "UV_LINK_MODE": "copy" }, + "forwardPorts": [8000, 8001, 9181], "customizations": { "vscode": { "extensions": [ diff --git a/projects/ecoindex_api/Taskfile.yml b/projects/ecoindex_api/Taskfile.yml index 6fc1102..6ddcc01 100644 --- a/projects/ecoindex_api/Taskfile.yml +++ b/projects/ecoindex_api/Taskfile.yml @@ -183,7 +183,7 @@ tasks: deps: [start-redis] cmds: - | - REDIS_DEV_HOST="$(uv run python scripts/wait_redis.py)" + REDIS_DEV_HOST="$(uv run python projects/ecoindex_api/scripts/wait_redis.py)" export REDIS_CACHE_HOST="$REDIS_DEV_HOST" export PYTHONPATH="bases:components" export RQ_WORKERS="{{.RQ_WORKERS}}" @@ -197,7 +197,7 @@ tasks: deps: [start-redis] cmds: - | - export REDIS_CACHE_HOST="$(uv run python scripts/wait_redis.py)" + export REDIS_CACHE_HOST="$(uv run python projects/ecoindex_api/scripts/wait_redis.py)" export PYTHONPATH="bases:components" uv run --package ecoindex_api --group backend uvicorn ecoindex.backend.main:app --host 0.0.0.0 --port 8000 --reload --reload-dir bases --reload-dir components dir: ../.. @@ -208,7 +208,7 @@ tasks: deps: [start-redis] cmds: - | - REDIS_DEV_HOST="$(uv run python scripts/wait_redis.py)" + REDIS_DEV_HOST="$(uv run python projects/ecoindex_api/scripts/wait_redis.py)" uv run --package ecoindex_api --group dev rq-dashboard \ --bind 0.0.0.0 \ --port {{.RQ_DASHBOARD_PORT}} \ diff --git a/projects/ecoindex_api/alembic/versions/5afa2faea43f_.py b/projects/ecoindex_api/alembic/versions/5afa2faea43f_.py index d07021c..79e2613 100644 --- a/projects/ecoindex_api/alembic/versions/5afa2faea43f_.py +++ b/projects/ecoindex_api/alembic/versions/5afa2faea43f_.py @@ -8,6 +8,7 @@ import sqlalchemy as sa import sqlmodel from alembic import op +from ecoindex.database.helper import column_exists revision = "5afa2faea43f" down_revision = "7eaafaa65b32" @@ -16,11 +17,13 @@ def upgrade() -> None: - op.add_column( - "apiecoindex", - sa.Column("source", sqlmodel.sql.sqltypes.AutoString(), nullable=True), # type: ignore - ) + if not column_exists(op.get_bind(), "apiecoindex", "source"): + op.add_column( + "apiecoindex", + sa.Column("source", sqlmodel.sql.sqltypes.AutoString(), nullable=True), # type: ignore + ) def downgrade() -> None: - op.drop_column("apiecoindex", "source") + if column_exists(op.get_bind(), "apiecoindex", "source"): + op.drop_column("apiecoindex", "source") diff --git a/projects/ecoindex_api/alembic/versions/e83263a5def4_add_index_id_and_host.py b/projects/ecoindex_api/alembic/versions/e83263a5def4_add_index_id_and_host.py index 73db901..224f7c5 100644 --- a/projects/ecoindex_api/alembic/versions/e83263a5def4_add_index_id_and_host.py +++ b/projects/ecoindex_api/alembic/versions/e83263a5def4_add_index_id_and_host.py @@ -21,7 +21,7 @@ def upgrade() -> None: with op.batch_alter_table("apiecoindex", schema=None) as batch_op: batch_op.alter_column( "id", - existing_type=sqlmodel.sql.sqltypes.GUID(), + existing_type=sa.Uuid(), nullable=False, ) batch_op.alter_column("version", existing_type=sa.INTEGER(), nullable=False) @@ -46,6 +46,6 @@ def downgrade() -> None: batch_op.alter_column("version", existing_type=sa.INTEGER(), nullable=True) batch_op.alter_column( "id", - existing_type=sqlmodel.sql.sqltypes.GUID(), + existing_type=sa.Uuid(), nullable=True, ) diff --git a/projects/ecoindex_api/alembic/versions/fd9a1f5662c8_first_migration.py b/projects/ecoindex_api/alembic/versions/fd9a1f5662c8_first_migration.py index 7545c2d..6dc6dc3 100644 --- a/projects/ecoindex_api/alembic/versions/fd9a1f5662c8_first_migration.py +++ b/projects/ecoindex_api/alembic/versions/fd9a1f5662c8_first_migration.py @@ -33,7 +33,7 @@ def upgrade() -> None: sa.Column("water", sa.Float(), nullable=True), sa.Column("date", sa.DateTime(), nullable=True), sa.Column("page_type", sqlmodel.sql.sqltypes.AutoString(), nullable=True), - sa.Column("id", sqlmodel.sql.sqltypes.GUID(), nullable=True), + sa.Column("id", sa.Uuid(), nullable=True), sa.Column("host", sqlmodel.sql.sqltypes.AutoString(), nullable=False), sa.Column("version", sa.Integer(), nullable=True), sa.Column("initial_ranking", sa.Integer(), nullable=False), diff --git a/projects/ecoindex_api/docker/worker/dockerfile b/projects/ecoindex_api/docker/worker/dockerfile index f961341..8823400 100644 --- a/projects/ecoindex_api/docker/worker/dockerfile +++ b/projects/ecoindex_api/docker/worker/dockerfile @@ -30,6 +30,7 @@ RUN rm -rf $wheel requirements.txt /tmp/dist /var/lib/{apt,dpkg,cache,log}/ COPY projects/ecoindex_api/docker/worker/entrypoint.sh /usr/bin/entrypoint COPY projects/ecoindex_api/scripts/start_rq_workers.sh /usr/bin/start_rq_workers.sh +COPY projects/ecoindex_api/scripts/clean_stale_rq_workers.py /usr/bin/clean_stale_rq_workers.py RUN chmod +x /usr/bin/entrypoint /usr/bin/start_rq_workers.sh ENTRYPOINT [ "/usr/bin/entrypoint" ] diff --git a/projects/ecoindex_api/pyproject.toml b/projects/ecoindex_api/pyproject.toml index 769c381..e01d42b 100644 --- a/projects/ecoindex_api/pyproject.toml +++ b/projects/ecoindex_api/pyproject.toml @@ -46,7 +46,7 @@ worker = [ ] dev = [ "aiosqlite>=0.19.0", - "rq-dashboard @ git+https://github.com/Parallels/rq-dashboard.git", + "rq-dashboard", "typing-extensions>=4.8.0", "watchdog>=6.0.0", ] diff --git a/projects/ecoindex_api/scripts/clean_stale_rq_workers.py b/projects/ecoindex_api/scripts/clean_stale_rq_workers.py new file mode 100644 index 0000000..a0244b4 --- /dev/null +++ b/projects/ecoindex_api/scripts/clean_stale_rq_workers.py @@ -0,0 +1,71 @@ +"""Unregister stale RQ workers before starting new worker processes.""" + +from __future__ import annotations + +import os +import time + +from redis import Redis +from redis.exceptions import ConnectionError as RedisConnectionError +from rq import Worker +from rq.utils import utcparse + +WORKER_NAME_PREFIX = "ecoindex-worker-" + + +def redis_url() -> str: + return os.environ.get( + "RQ_REDIS_URL", + f"redis://{os.environ.get('REDIS_CACHE_HOST', 'localhost')}:6379/0", + ) + + +def worker_host_id() -> str: + return os.environ.get("RQ_WORKER_HOST") or os.environ.get("HOSTNAME", "local") + + +def planned_worker_names() -> set[str]: + worker_count = int(os.environ.get("RQ_WORKERS", "3")) + host_id = worker_host_id() + names = {f"{WORKER_NAME_PREFIX}{host_id}-{index}" for index in range(1, worker_count + 1)} + # Legacy fixed names kept during transition from older deployments. + names.update(f"{WORKER_NAME_PREFIX}{index}" for index in range(1, worker_count + 1)) + return names + + +def is_stale(worker: Worker, max_age_seconds: int) -> bool: + if worker.last_heartbeat is None: + return True + + heartbeat = worker.last_heartbeat + if isinstance(heartbeat, str): + heartbeat = utcparse(heartbeat) + + return (time.time() - heartbeat.timestamp()) > max_age_seconds + + +def cleanup_stale_workers(connection: Redis, max_age_seconds: int) -> None: + planned_names = planned_worker_names() + + for worker in Worker.all(connection=connection): + if not worker.name.startswith(WORKER_NAME_PREFIX): + continue + + if worker.name not in planned_names and not is_stale(worker, max_age_seconds): + continue + + worker.register_death() + + +def main() -> None: + max_age_seconds = int(os.environ.get("RQ_STALE_WORKER_SECONDS", "60")) + + try: + connection = Redis.from_url(redis_url()) + cleanup_stale_workers(connection, max_age_seconds) + except RedisConnectionError as exc: + raise SystemExit(f"Redis is not reachable: {exc}") from exc + + +if __name__ == "__main__": + main() diff --git a/projects/ecoindex_api/scripts/start_rq_workers.sh b/projects/ecoindex_api/scripts/start_rq_workers.sh index 49c89ba..6b6ed65 100755 --- a/projects/ecoindex_api/scripts/start_rq_workers.sh +++ b/projects/ecoindex_api/scripts/start_rq_workers.sh @@ -3,6 +3,16 @@ set -e RQ_WORKERS="${RQ_WORKERS:-3}" REDIS_URL="${RQ_REDIS_URL:-redis://${REDIS_CACHE_HOST:-localhost}:6379/0}" +HOST_ID="${RQ_WORKER_HOST:-${HOSTNAME:-local}}" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" + +export RQ_WORKERS RQ_REDIS_URL="$REDIS_URL" RQ_WORKER_HOST="$HOST_ID" + +if [ "${USE_UV:-0}" = "1" ]; then + uv run python "$SCRIPT_DIR/clean_stale_rq_workers.py" +else + python "$SCRIPT_DIR/clean_stale_rq_workers.py" +fi start_worker() { if [ "${USE_UV:-0}" = "1" ]; then @@ -16,7 +26,7 @@ i=1 while [ "$i" -le "$RQ_WORKERS" ]; do start_worker ecoindex ecoindex_batch \ --url "$REDIS_URL" \ - --name "ecoindex-worker-$i" & + --name "ecoindex-worker-${HOST_ID}-$i" & i=$((i + 1)) done diff --git a/tasks/DockerTaskfile.yml b/tasks/DockerTaskfile.yml index 9ca48c3..1967621 100644 --- a/tasks/DockerTaskfile.yml +++ b/tasks/DockerTaskfile.yml @@ -4,8 +4,14 @@ tasks: build: internal: true cmds: - - echo "docker build -t {{.NAME}} {{.VERSION}} {{.OPTIONS}} ." - - docker build -t vvatelot/ecoindex-{{.NAME}}:{{.VERSION}} -t vvatelot/ecoindex-{{.NAME}}:latest {{.OPTIONS}} . + - | + if docker buildx version > /dev/null 2>&1; then + echo "docker buildx build --load -t vvatelot/ecoindex-{{.NAME}}:{{.VERSION}} -t vvatelot/ecoindex-{{.NAME}}:latest {{.OPTIONS}} ." + docker buildx build --load -t vvatelot/ecoindex-{{.NAME}}:{{.VERSION}} -t vvatelot/ecoindex-{{.NAME}}:latest {{.OPTIONS}} . + else + echo "docker build -t vvatelot/ecoindex-{{.NAME}}:{{.VERSION}} -t vvatelot/ecoindex-{{.NAME}}:latest {{.OPTIONS}} ." + docker build -t vvatelot/ecoindex-{{.NAME}}:{{.VERSION}} -t vvatelot/ecoindex-{{.NAME}}:latest {{.OPTIONS}} . + fi silent: true push: diff --git a/uv.lock b/uv.lock index 0ae1a7b..b0151a3 100644 --- a/uv.lock +++ b/uv.lock @@ -720,7 +720,7 @@ provides-extras = ["webp"] backend = [{ name = "uvicorn", specifier = ">=0.23.2" }] dev = [ { name = "aiosqlite", specifier = ">=0.19.0" }, - { name = "rq-dashboard", git = "https://github.com/Parallels/rq-dashboard.git" }, + { name = "rq-dashboard" }, { name = "typing-extensions", specifier = ">=4.8.0" }, { name = "watchdog", specifier = ">=6.0.0" }, ] @@ -2333,7 +2333,7 @@ wheels = [ [[package]] name = "rq-dashboard" version = "0.9.0" -source = { git = "https://github.com/Parallels/rq-dashboard.git#3735f1ebb6a936ff5ad1ef6b3a3bfae58bc03029" } +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "arrow" }, { name = "flask" }, @@ -2341,6 +2341,10 @@ dependencies = [ { name = "redis-sentinel-url" }, { name = "rq" }, ] +sdist = { url = "https://files.pythonhosted.org/packages/ea/46/f5bae00d7b29a90bd88158b25757c3624073e0e766c0d82bc1325a827a7f/rq_dashboard-0.9.0.tar.gz", hash = "sha256:33472117abe47b793d608e60fff5d29ed84e8e5a2c13c793ea3dbfdbde52dfb6", size = 214248, upload-time = "2026-07-02T08:02:20.577Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/83/2f399fa789778dc8f3f4b5e74c23b06d736135e3cb93c1369b88dc431468/rq_dashboard-0.9.0-py2.py3-none-any.whl", hash = "sha256:bcb670a097b1539d1cfe752a1609e3870177b253c4ef49e037d518eef2cd8b86", size = 217225, upload-time = "2026-07-02T08:02:18.879Z" }, +] [[package]] name = "ruff" From 5235c269fe14302ace79ee449c1bde0580734492 Mon Sep 17 00:00:00 2001 From: Vincent Vatelot Date: Tue, 7 Jul 2026 13:04:34 +0000 Subject: [PATCH 5/5] fix(api): update worker type in stale check function - Changed the parameter type of the `is_stale` function from `Worker` to `BaseWorker` to align with the updated RQ library structure. --- projects/ecoindex_api/scripts/clean_stale_rq_workers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/ecoindex_api/scripts/clean_stale_rq_workers.py b/projects/ecoindex_api/scripts/clean_stale_rq_workers.py index a0244b4..95077b7 100644 --- a/projects/ecoindex_api/scripts/clean_stale_rq_workers.py +++ b/projects/ecoindex_api/scripts/clean_stale_rq_workers.py @@ -9,6 +9,7 @@ from redis.exceptions import ConnectionError as RedisConnectionError from rq import Worker from rq.utils import utcparse +from rq.worker.base import BaseWorker WORKER_NAME_PREFIX = "ecoindex-worker-" @@ -33,7 +34,7 @@ def planned_worker_names() -> set[str]: return names -def is_stale(worker: Worker, max_age_seconds: int) -> bool: +def is_stale(worker: BaseWorker, max_age_seconds: int) -> bool: if worker.last_heartbeat is None: return True