UN-2813 Keep only necessary changes

Author: Deepak-Kesavan

task-backend/Dockerfile

@@ -0,0 +1,50 @@
+# Task Backend Worker - Production Dockerfile
+FROM python:3.12-slim
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1
+
+# Create non-root user
+RUN groupadd -r taskworker && useradd -r -g taskworker taskworker
+
+# Install system dependencies
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /app
+
+# Copy requirements and install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy task-abstraction library
+COPY unstract/task-abstraction/src /app/task-abstraction/src
+ENV PYTHONPATH="/app/task-abstraction/src:$PYTHONPATH"
+
+# Copy task-backend service
+COPY task-backend/src /app/src
+COPY task-backend/pyproject.toml /app/
+
+# Install task-backend in editable mode
+RUN pip install -e .
+
+# Create directories for logs and data
+RUN mkdir -p /app/logs /app/data \
+    && chown -R taskworker:taskworker /app
+
+# Switch to non-root user
+USER taskworker
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
+    CMD python -m unstract.task_backend.cli.main --health-check
+
+# Default command
+CMD ["python", "-m", "unstract.task_backend.cli.main"]

task-backend/README.md

@@ -0,0 +1,83 @@
+# Task Backend Worker Service
+
+A worker management service that uses the `unstract-task-abstraction` library to initialize and run backend-specific workers.
+
+## Architecture
+
+This service is **NOT an HTTP service** - it's a **worker process** that:
+
+1. **Uses task-abstraction library** to get the appropriate backend client
+2. **Initializes backend-specific workers** (Celery, Hatchet, Temporal)
+3. **Registers workflows** with the selected backend
+4. **Runs workers** that consume tasks from the backend
+
+## Backend Types
+
+### Celery Workers
+- Connect to Redis broker
+- Start multiple worker processes
+- Consume from configured queues
+- Similar to existing `backend/workers/` pattern
+
+### Hatchet Workers
+- Connect to Hatchet server via HTTP/gRPC
+- Register workflows with server
+- Poll for task assignments
+- Execute tasks and report results
+
+### Temporal Workers
+- Connect to Temporal server via gRPC
+- Register activities and workflows
+- Listen for workflow executions
+- Handle activity execution
+
+## Usage
+
+```bash
+# Install dependencies
+uv sync
+
+# Copy configuration
+cp sample.env .env
+# Edit .env to set TASK_BACKEND_TYPE and connection details
+
+# Start worker (auto-detects backend from env)
+task-backend-worker
+
+# Or start specific backend
+task-backend-worker --backend=celery
+task-backend-worker --backend=hatchet
+task-backend-worker --backend=temporal
+```
+
+## Configuration
+
+See `sample.env` for all configuration options. Key settings:
+
+- `TASK_BACKEND_TYPE`: Backend to use (celery, hatchet, temporal)
+- `CELERY_BROKER_URL`: Redis URL for Celery
+- `HATCHET_CLIENT_TOKEN`: Token for Hatchet server
+- `TEMPORAL_HOST`: Temporal server host
+
+## Integration with Platform
+
+This service replaces the functionality of:
+- Runner Service workers
+- Prompt Service workers
+- Structure Tool workers
+
+It provides the same task execution capabilities but through a unified, backend-agnostic interface using the task-abstraction library.
+
+## Development
+
+```bash
+# Run tests
+uv run pytest
+
+# Linting
+uv run ruff check .
+uv run ruff format .
+
+# Type checking
+uv run mypy .
+```

task-backend/docker-compose.yml

@@ -0,0 +1,148 @@
+# Task Backend Worker - Docker Compose for development and testing
+version: '3.8'
+
+services:
+  # Redis for Celery backend
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    command: redis-server --appendonly yes
+    volumes:
+      - redis_data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+
+  # Task worker with Celery backend
+  worker-celery:
+    build: .
+    environment:
+      - TASK_BACKEND_TYPE=celery
+      - TASK_CELERY_BROKER_URL=redis://redis:6379/0
+      - TASK_CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - TASK_WORKER_NAME=celery-worker-1
+      - TASK_WORKER_CONCURRENCY=4
+      - TASK_LOG_LEVEL=INFO
+    depends_on:
+      redis:
+        condition: service_healthy
+    volumes:
+      - ./logs:/app/logs
+    restart: unless-stopped
+
+  # Task worker with Hatchet backend (requires external Hatchet server)
+  worker-hatchet:
+    build: .
+    environment:
+      - TASK_BACKEND_TYPE=hatchet
+      - TASK_HATCHET_TOKEN=${HATCHET_TOKEN}
+      - TASK_HATCHET_SERVER_URL=${HATCHET_SERVER_URL:-https://app.hatchet.run}
+      - TASK_WORKER_NAME=hatchet-worker-1
+      - TASK_LOG_LEVEL=INFO
+    volumes:
+      - ./logs:/app/logs
+    restart: unless-stopped
+    profiles:
+      - hatchet
+
+  # Temporal server for development
+  temporal:
+    image: temporalio/auto-setup:1.22
+    ports:
+      - "7233:7233"
+      - "8080:8080"  # Web UI
+    environment:
+      - DB=postgresql
+      - DB_PORT=5432
+      - POSTGRES_USER=temporal
+      - POSTGRES_PWD=temporal
+      - POSTGRES_SEEDS=postgres
+    depends_on:
+      postgres:
+        condition: service_healthy
+    volumes:
+      - temporal_data:/etc/temporal
+    profiles:
+      - temporal
+
+  # PostgreSQL for Temporal
+  postgres:
+    image: postgres:15-alpine
+    environment:
+      - POSTGRES_USER=temporal
+      - POSTGRES_PASSWORD=temporal
+      - POSTGRES_DB=temporal
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U temporal"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    profiles:
+      - temporal
+
+  # Task worker with Temporal backend
+  worker-temporal:
+    build: .
+    environment:
+      - TASK_BACKEND_TYPE=temporal
+      - TASK_TEMPORAL_HOST=temporal
+      - TASK_TEMPORAL_PORT=7233
+      - TASK_TEMPORAL_NAMESPACE=default
+      - TASK_TEMPORAL_TASK_QUEUE=task-queue
+      - TASK_WORKER_NAME=temporal-worker-1
+      - TASK_LOG_LEVEL=INFO
+    depends_on:
+      temporal:
+        condition: service_started
+    volumes:
+      - ./logs:/app/logs
+    restart: unless-stopped
+    profiles:
+      - temporal
+
+  # Monitoring and observability
+  prometheus:
+    image: prom/prometheus:latest
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+    profiles:
+      - monitoring
+
+  grafana:
+    image: grafana/grafana:latest
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards
+      - ./monitoring/grafana/datasources:/etc/grafana/provisioning/datasources
+    profiles:
+      - monitoring
+
+volumes:
+  redis_data:
+  postgres_data:
+  temporal_data:
+  prometheus_data:
+  grafana_data:
+
+networks:
+  default:
+    name: task-backend-network