Switch to Valkey, drop Celery (#18)

Co-authored-by: Filip Vágner <therazix@gmail.com>

Author: martinhoyer

.github/workflows/test.yml

@@ -42,23 +42,16 @@ jobs:
     - name: Create Podman pod
       run: |
         podman pod create --name tmt-web-pod --infra-image=registry.k8s.io/pause:3.9 -p 8000:8000 -p 6379:6379
-        # Exposing redis port as well for test_api.py::TestCelery::test_basic_test_request
+        # Exposing valkey port as well for background task tests
 
-    - name: Start Redis container
+    - name: Start valkey container
       run: |
-        podman run -d --pod tmt-web-pod --name redis redis:latest
-
-    - name: Start Celery container
-      run: |
-        podman run -d --pod tmt-web-pod --name celery \
-          -e REDIS_URL=redis://localhost:6379 \
-          -e API_HOSTNAME=http://localhost:8000 \
-          tmt-web:latest celery --app=tmt_web.service worker --loglevel=INFO
+        podman run -d --pod tmt-web-pod --name valkey valkey/valkey:alpine
 
     - name: Start Web container
       run: |
         podman run -d --pod tmt-web-pod --name web \
-          -e REDIS_URL=redis://localhost:6379 \
+          -e VALKEY_URL=valkey://localhost:6379 \
           -e API_HOSTNAME=http://localhost:8000 \
           tmt-web:latest uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000
 

.pre-commit-config.yaml

@@ -24,7 +24,6 @@ repos:
         additional_dependencies:
           - 'tmt'
           - 'pydantic'
-          - 'celery-types'
         pass_filenames: false
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit

README.md

@@ -12,10 +12,10 @@ podman-compose up --build
 
 Add `-d` for the service to run in the background.
 
-In order to quickly experiment without using Celery use this:
+For quick development without container setup:
 
 ```bash
-USE_CELERY=false CLONE_DIR_PATH=/var/tmp/test uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000
+CLONE_DIR_PATH=/var/tmp/test uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000
 ```
 
 ## Tests
@@ -29,15 +29,19 @@ Run `hatch env show` to see the list of available environments and their scripts
 
 ## Environment variables
 
-- `REDIS_URL` - *optional*, passed to Celery on initialization as a `broker` and
-  `backend` argument, default value is: `redis://localhost:6379`
+- `VALKEY_URL` - *optional*, connection URL for Valkey which is used for storing task state,
+  default value is: `valkey://localhost:6379`
 - `CLONE_DIR_PATH` - *optional*, specifies the path where the repositories will
   be cloned, default value is: `./.repos/`
-- `USE_CELERY` - *optional*, specifies if the app should use Celery, set to
-  `false` for running without Celery
 - `API_HOSTNAME` - *required*, specifies the hostname of the API, used for
   creating the callback URL to the service
 
+## Architecture
+
+The application uses FastAPI's built-in background tasks for asynchronous processing and
+Valkey for task state storage. This architecture provides a lightweight and efficient
+solution for handling long-running tasks without requiring external task queue infrastructure.
+
 ## API
 
 The API version is defined by prefix in url, e.g. `/v0.1/status`.
@@ -50,7 +54,7 @@ exclusive.
 
 ### `/`
 
-Returns ID of the created Celery task with additional metadata in JSON
+Returns ID of the created background task with additional metadata in JSON
 and callback url for `/status` endpoint, returns the same in HTML format
 if `format` is set to `html`.
 

compose.yaml

@@ -4,9 +4,8 @@ services:
     build:
       context: .
       dockerfile: ./Containerfile
-    command: uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000
     environment:
-      - REDIS_URL=redis://redis:6379
+      - VALKEY_URL=valkey://valkey:6379
       - API_HOSTNAME=http://localhost:8000
     ports:
       - 8000:8000
@@ -18,31 +17,17 @@ services:
       start_period: 5s
     restart: unless-stopped
     depends_on:
-      redis:
+      valkey:
         condition: service_healthy
 
-  redis:
-    container_name: redis
-    image: redis:latest
+  valkey:
+    container_name: valkey
+    image: valkey/valkey:alpine
     ports:
       - 6379:6379
     healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
+      test: ["CMD", "valkey-cli", "ping"]
       interval: 10s
       timeout: 5s
       retries: 3
     restart: unless-stopped
-
-  celery:
-    container_name: celery
-    build:
-      context: .
-      dockerfile: ./Containerfile
-    command: celery --app=tmt_web.service worker --loglevel=INFO
-    environment:
-      - REDIS_URL=redis://redis:6379
-      - API_HOSTNAME=http://localhost:8000
-    depends_on:
-      redis:
-        condition: service_healthy
-    restart: unless-stopped

entrypoint.sh

@@ -8,24 +8,8 @@ error() {
 # Handle signals
 trap 'kill -TERM $PID' TERM INT
 
-# Name of container to start
-APP=$1
+uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000 &
 
-[ -z "$APP" ] && error "No app to run passed to entrypoint script"
-
-case $APP in
-    uvicorn)
-        COMMAND="uvicorn tmt_web.api:app --reload --host 0.0.0.0 --port 8000"
-        ;;
-    celery)
-        COMMAND="celery --app=tmt_web.service worker --loglevel=INFO"
-        ;;
-    *)
-        error "Unknown app '$APP'"
-        ;;
-esac
-
-$COMMAND &
 PID=$!
 
 wait $PID

pyproject.toml

@@ -19,7 +19,6 @@ classifiers = [
   "Programming Language :: Python",
   "Programming Language :: Python :: 3.12",
   "Framework :: FastAPI",
-  "Framework :: Celery",
   "Topic :: Software Development :: Testing",
   "Operating System :: POSIX :: Linux",
 ]
@@ -28,7 +27,7 @@ dependencies = [
   "fastapi~=0.115",
   "httpx~=0.27",
   "uvicorn~=0.30",
-  "celery[redis]~=5.4",
+  "valkey",
 ]
 
 [project.urls]
@@ -134,7 +133,10 @@ lint.ignore = [
     "S101",  # Assert usage
     "PLR",  # Pylint refactor
     "E501",  # Line length
-    ]
+]
+"src/tmt_web/generators/html_generator.py" = [
+    "E501",  # Line length
+]
 
 [tool.ruff.lint.pydocstyle]
 convention = "pep257"
@@ -156,3 +158,15 @@ no_implicit_reexport = true
 
 python_version = "3.12"
 files = ["src/"]
+
+[tool.pytest.ini_options]
+# Filter out specific warnings
+filterwarnings = [
+    # Ignore pytest collection warnings for model classes that start with 'Test'
+    "ignore:cannot collect test class 'TestData' because it has a __init__ constructor:pytest.PytestCollectionWarning",
+    "ignore:cannot collect test class 'TestPlanData' because it has a __init__ constructor:pytest.PytestCollectionWarning",
+    # Ignore pint and docutils deprecations from tmt
+    "ignore:The frontend.OptionParser class will be replaced:DeprecationWarning",
+    "ignore:The frontend.Option class will be removed:DeprecationWarning",
+    "ignore:This function will be removed in future versions of pint.:DeprecationWarning"
+]