docker-compose.yaml

x-backend-secrets: &backend-secrets
  SECRET_KEY: ${SECRET_KEY:-CHANGE_ME_min_32_chars_long_!!!!}
  MONGO_USER: ${MONGO_USER:-root}
  MONGO_PASSWORD: ${MONGO_PASSWORD:-rootpassword}
  REDIS_PASSWORD: ${REDIS_PASSWORD:-redispassword}

services:
  # Shared base image for all Python backend services
  base:
    image: ghcr.io/hardmax71/integr8scode/base:${IMAGE_TAG:-latest}
    build:
      context: ./backend
      dockerfile: Dockerfile.base

  shared-ca:
    image: alpine:latest
    volumes:
      - shared_ca:/shared_ca
    command: sh -c "mkdir -p /shared_ca && chmod 777 /shared_ca && echo 'Shared CA directory ready'"
    networks:
      - app-network

  cert-generator:
    image: ghcr.io/hardmax71/integr8scode/cert-generator:${IMAGE_TAG:-latest}
    build:
      context: ./cert-generator
      dockerfile: Dockerfile
    volumes:
      - ./backend/certs:/backend-certs
      - ./frontend/certs:/frontend-certs
      - ~/.kube:/root/.kube
      - /etc/rancher/k3s:/etc/rancher/k3s:ro
      - shared_ca:/shared_ca
      - ./backend:/backend
    environment:
       - SHARED_CA_DIR=/shared_ca
       - BACKEND_CERT_DIR=/backend-certs
       - FRONTEND_CERT_DIR=/frontend-certs
    extra_hosts:
      - "host.docker.internal:host-gateway"
    restart: "no"
    networks:
      - app-network
    depends_on:
      shared-ca:
        condition: service_completed_successfully

  mongo:
    image: mongo:8.0.17
    command: ["mongod", "--wiredTigerCacheSizeGB", "0.4"]
    ports:
      - "127.0.0.1:27017:27017"
    environment:
      MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER:-root}
      MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD:-rootpassword}
      MONGO_INITDB_DATABASE: integr8scode
    volumes:
      - mongo_data:/data/db
    networks:
      - app-network
    container_name: mongo
    mem_limit: 1024m
    restart: unless-stopped
    ulimits:
      nofile:
        soft: 65536
        hard: 65536
    healthcheck:
      test: mongosh --quiet --host 127.0.0.1 --port 27017 -u ${MONGO_USER:-root} -p ${MONGO_PASSWORD:-rootpassword} --authenticationDatabase admin --eval 'db.adminCommand("ping")'
      interval: 3s
      timeout: 5s
      retries: 15
      start_period: 5s

  redis:
    image: redis:7.4.6-alpine
    container_name: redis
    ports:
      - "127.0.0.1:6379:6379"
    command: redis-server --maxmemory 256mb --maxmemory-policy allkeys-lru --save "" --requirepass ${REDIS_PASSWORD:-redispassword}
    volumes:
      - redis_data:/data
    networks:
      - app-network
    mem_limit: 300m
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD:-redispassword}", "--no-auth-warning", "ping"]
      interval: 2s
      timeout: 3s
      retries: 10
      start_period: 2s

  backend:
    image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest}
    build:
      context: ./backend
      dockerfile: Dockerfile
      additional_contexts:
        base: service:base
    depends_on:
      base:
        condition: service_completed_successfully
      cert-generator:
        condition: service_completed_successfully
      user-seed:
        condition: service_completed_successfully
      mongo:
        condition: service_healthy
      redis:
        condition: service_healthy
      kafka:
        condition: service_healthy
    volumes:
      - ./backend/app:/app/app
      - ./backend/workers:/app/workers
      - ./backend/scripts:/app/scripts
      - ./backend/tests:/app/tests:ro
      - ./backend/certs:/app/certs:ro
      - ./backend/config.test.toml:/app/config.test.toml:ro
      - ./backend/config.toml:/app/config.toml:ro
      - shared_ca:/shared_ca:ro
      - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro
    ports:
      - "443:443"
      - "127.0.0.1:9090:9090"  # Metrics port (host:container)
    networks:
      - app-network
    container_name: backend
    environment:
      <<: *backend-secrets
      WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2}
      WEB_THREADS: ${WEB_THREADS:-4}
      WEB_TIMEOUT: ${WEB_TIMEOUT:-60}
    extra_hosts:
      - "host.docker.internal:host-gateway"
    mem_limit: 768m
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "curl -k -f -s https://localhost:443/api/v1/health/live >/dev/null || exit 1"]
      interval: 2s
      timeout: 3s
      retries: 30
      start_period: 3s

  frontend:
    image: ghcr.io/hardmax71/integr8scode/frontend:${IMAGE_TAG:-latest}
    container_name: frontend
    build:
      context: ./frontend
      dockerfile: Dockerfile
    depends_on:
      cert-generator:
        condition: service_completed_successfully
      backend:
        condition: service_started
    volumes:
      - ./frontend/certs:/etc/nginx/certs:ro
      - shared_ca:/shared_ca:ro
    ports:
      - "5001:5001"
    networks:
      - app-network
    environment:
      - BACKEND_URL=https://backend:443
      - GRAFANA_URL=${GRAFANA_URL:-http://grafana:3000}
    mem_limit: 128m
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "curl -k -f -s https://localhost:5001 >/dev/null || exit 1"]
      interval: 2s
      timeout: 3s
      retries: 30
      start_period: 3s


  grafana:
    container_name: grafana
    profiles: ["observability"]
    image: grafana/grafana:12.3.1
    user: "472"
    entrypoint:
      - sh
      - -c
      - |
        grafana-cli --homepath /usr/share/grafana admin reset-admin-password "$$GF_SECURITY_ADMIN_PASSWORD" > /dev/null 2>&1 || true
        exec /run.sh
    ports:
      - "127.0.0.1:3000:3000"
    volumes:
      - ./backend/grafana/grafana.ini:/etc/grafana/grafana.ini:ro
      - ./backend/grafana/provisioning:/etc/grafana/provisioning:ro
      - grafana_data:/var/lib/grafana
    networks:
      - app-network
    mem_limit: 192m
    restart: unless-stopped
    depends_on:
      victoria-metrics:
        condition: service_healthy
      jaeger:
        condition: service_started
      loki:
        condition: service_healthy
    environment:
      - GF_LOG_LEVEL=warn
      - GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL:-http://localhost:3000/}
      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin}
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:-admin123}
      - GF_SMTP_HOST=${GF_SMTP_HOST:-}
      - GF_SMTP_USER=${MAILJET_API_KEY:-}
      - GF_SMTP_PASSWORD=${MAILJET_SECRET_KEY:-}
      - GF_SMTP_FROM_ADDRESS=${MAILJET_FROM_ADDRESS:-noreply@integr8scode.cc}
      - GF_SMTP_ALERT_RECIPIENTS=${GRAFANA_ALERT_RECIPIENTS:-noreply@integr8scode.cc}
    healthcheck:
      test: ["CMD-SHELL", "curl -f localhost:3000/api/health && echo 'ready'"]
      interval: 10s
      timeout: 5s
      retries: 10
      start_period: 30s

  # Kafka (KRaft mode — no ZooKeeper)
  kafka:
    image: confluentinc/cp-kafka:7.8.2
    container_name: kafka
    ports:
      - "127.0.0.1:9092:9092"
      - "127.0.0.1:29092:29092"
    environment:
      # KRaft identity (combined broker + controller)
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: 'broker,controller'
      CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'

      # Listeners
      KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'

      # Single-node replication
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1

      # Topic management
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_DELETE_TOPIC_ENABLE: 'true'
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_LOG_RETENTION_HOURS: 168

      # Production settings
      KAFKA_COMPRESSION_TYPE: 'gzip'
      KAFKA_NUM_NETWORK_THREADS: 8
      KAFKA_NUM_IO_THREADS: 8
      KAFKA_SOCKET_SEND_BUFFER_BYTES: 102400
      KAFKA_SOCKET_RECEIVE_BUFFER_BYTES: 102400
      KAFKA_SOCKET_REQUEST_MAX_BYTES: 104857600

      # Log settings
      KAFKA_LOG_SEGMENT_BYTES: 1073741824
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 300000
      KAFKA_LOG_CLEANUP_POLICY: 'delete'

      # JVM settings
      KAFKA_HEAP_OPTS: '-Xms256m -Xmx1g'
      KAFKA_JVM_PERFORMANCE_OPTS: '-server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -Djava.awt.headless=true'
    volumes:
      - kafka_data:/var/lib/kafka/data
      - kafka_logs:/var/log/kafka
    networks:
      - app-network
    ulimits:
      nofile:
        soft: 65536
        hard: 65536
    mem_limit: 1280m
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:9092"]
      interval: 3s
      timeout: 10s
      retries: 15
      start_period: 3s

  kafdrop:
    image: obsidiandynamics/kafdrop:3.31.0
    container_name: kafdrop
    profiles: ["debug"]
    depends_on:
      - kafka
    ports:
      - "127.0.0.1:9000:9000"
    environment:
      KAFKA_BROKERCONNECT: kafka:29092
      JVM_OPTS: "-Xms256M -Xmx512M"
    networks:
      - app-network

  # Seed default users (runs once after mongo is ready)
  user-seed:
    image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest}
    container_name: user-seed
    depends_on:
      mongo:
        condition: service_healthy
    environment:
      <<: *backend-secrets
      DEFAULT_USER_PASSWORD: ${DEFAULT_USER_PASSWORD:-user123}
      ADMIN_USER_PASSWORD: ${ADMIN_USER_PASSWORD:-admin123}
    volumes:
      - ./backend/app:/app/app:ro
      - ./backend/scripts:/app/scripts:ro
      - ./backend/config.toml:/app/config.toml:ro
    command: ["python", "-m", "scripts.seed_users"]
    networks:
      - app-network
    restart: "no"  # Run once and exit

  # Event-driven workers
  k8s-worker:
    image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest}
    container_name: k8s-worker
    mem_limit: 160m
    command: ["python", "workers/run_k8s_worker.py"]
    depends_on:
      kafka:
        condition: service_healthy
      mongo:
        condition: service_started
    environment:
      <<: *backend-secrets
    volumes:
      - ./backend/app:/app/app:ro
      - ./backend/workers:/app/workers:ro
      - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro
      - ./backend/config.toml:/app/config.toml:ro
      - ./backend/config.k8s-worker.toml:/app/config.k8s-worker.toml:ro
    networks:
      - app-network
    extra_hosts:
      - "host.docker.internal:host-gateway"
    restart: unless-stopped

  pod-monitor:
    image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest}
    container_name: pod-monitor
    mem_limit: 160m
    command: ["python", "workers/run_pod_monitor.py"]
    depends_on:
      kafka:
        condition: service_healthy
    environment:
      <<: *backend-secrets
    volumes:
      - ./backend/app:/app/app:ro
      - ./backend/workers:/app/workers:ro
      - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro
      - ./backend/config.toml:/app/config.toml:ro
      - ./backend/config.pod-monitor.toml:/app/config.pod-monitor.toml:ro
    networks:
      - app-network
    extra_hosts:
      - "host.docker.internal:host-gateway"
    restart: unless-stopped

  result-processor:
    image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest}
    container_name: result-processor
    mem_limit: 160m
    command: ["python", "workers/run_result_processor.py"]
    depends_on:
      kafka:
        condition: service_healthy
      mongo:
        condition: service_started
    environment:
      <<: *backend-secrets
    volumes:
      - ./backend/app:/app/app:ro
      - ./backend/workers:/app/workers:ro
      - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro
      - ./backend/config.toml:/app/config.toml:ro
      - ./backend/config.result-processor.toml:/app/config.result-processor.toml:ro
    networks:
      - app-network
    extra_hosts:
      - "host.docker.internal:host-gateway"
    restart: unless-stopped

  saga-orchestrator:
    image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest}
    container_name: saga-orchestrator
    mem_limit: 160m
    command: ["python", "workers/run_saga_orchestrator.py"]
    depends_on:
      kafka:
        condition: service_healthy
      mongo:
        condition: service_started
    environment:
      <<: *backend-secrets
    volumes:
      - ./backend/app:/app/app:ro
      - ./backend/workers:/app/workers:ro
      - ./backend/config.toml:/app/config.toml:ro
      - ./backend/config.saga-orchestrator.toml:/app/config.saga-orchestrator.toml:ro
    networks:
      - app-network
    restart: unless-stopped

  # Distributed tracing with Jaeger
  jaeger:
    image: jaegertracing/all-in-one:1.52
    container_name: jaeger
    profiles: ["observability"]
    mem_limit: 256m
    ports:
      - "127.0.0.1:5775:5775/udp"   # Zipkin/thrift compact
      - "127.0.0.1:6831:6831/udp"   # Thrift compact
      - "127.0.0.1:6832:6832/udp"   # Thrift binary
      - "127.0.0.1:5778:5778"       # HTTP config
      - "127.0.0.1:16686:16686"     # Jaeger UI
      - "127.0.0.1:14268:14268"     # HTTP collector
      - "127.0.0.1:14250:14250"     # gRPC collector
      - "127.0.0.1:9411:9411"       # Zipkin compatible endpoint
    environment:
      - COLLECTOR_ZIPKIN_HOST_PORT=:9411
      - COLLECTOR_OTLP_ENABLED=true
    networks:
      - app-network

  loki:
    container_name: loki
    profiles: ["observability"]
    image: grafana/loki:3.4.2
    ports:
      - "127.0.0.1:3100:3100"
    volumes:
      - ./backend/loki/loki-config.yaml:/etc/loki/local-config.yaml:ro
      - loki_data:/loki
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - app-network
    mem_limit: 256m
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "wget --quiet --tries=1 --output-document=- http://localhost:3100/ready | grep -q -w ready || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 12
      start_period: 20s

  # Event replay service
  event-replay:
    image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest}
    container_name: event-replay
    mem_limit: 160m
    command: ["python", "workers/run_event_replay.py"]
    depends_on:
      kafka:
        condition: service_healthy
      mongo:
        condition: service_started
    environment:
      <<: *backend-secrets
    volumes:
      - ./backend/app:/app/app:ro
      - ./backend/workers:/app/workers:ro
      - ./backend/config.toml:/app/config.toml:ro
      - ./backend/config.event-replay.toml:/app/config.event-replay.toml:ro
    networks:
      - app-network
    restart: unless-stopped

  # DLQ Processor Service
  dlq-processor:
    image: ghcr.io/hardmax71/integr8scode/backend:${IMAGE_TAG:-latest}
    container_name: dlq-processor
    mem_limit: 160m
    command: ["python", "workers/run_dlq_processor.py"]
    depends_on:
      kafka:
        condition: service_healthy
      mongo:
        condition: service_started
    environment:
      <<: *backend-secrets
    volumes:
      - ./backend/app:/app/app:ro
      - ./backend/workers:/app/workers:ro
      - ./backend/config.toml:/app/config.toml:ro
      - ./backend/config.dlq-processor.toml:/app/config.dlq-processor.toml:ro
    networks:
      - app-network
    restart: unless-stopped

  # Monitoring Stack
  # Victoria Metrics - Time series database
  victoria-metrics:
    image: victoriametrics/victoria-metrics:v1.96.0
    container_name: victoria-metrics
    profiles: ["observability"]
    mem_limit: 256m
    ports:
      - "127.0.0.1:8428:8428"
    volumes:
      - victoria_metrics_data:/victoria-metrics-data
    command:
      - "--storageDataPath=/victoria-metrics-data"
      - "--httpListenAddr=0.0.0.0:8428"
      - "--retentionPeriod=30d"
    networks:
      - app-network
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8428/-/healthy"]
      interval: 5s
      timeout: 5s
      retries: 12
      start_period: 10s

  # OpenTelemetry Collector
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.91.0
    container_name: otel-collector
    profiles: ["observability"]
    mem_limit: 192m
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./backend/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
      - /var/run/docker.sock:/var/run/docker.sock:ro
      - /proc:/hostfs/proc:ro
      - /sys:/hostfs/sys:ro
    environment:
      - HOST_PROC=/hostfs/proc
      - HOST_SYS=/hostfs/sys
      - MONGO_USER=${MONGO_USER:-root}
      - MONGO_PASSWORD=${MONGO_PASSWORD:-rootpassword}
    ports:
      - "127.0.0.1:4317:4317"   # OTLP gRPC
      - "127.0.0.1:4318:4318"   # OTLP HTTP
      - "127.0.0.1:13133:13133" # Health check
    networks:
      - app-network
    depends_on:
      victoria-metrics:
        condition: service_healthy
      jaeger:
        condition: service_started
      loki:
        condition: service_healthy
      kafka:
        condition: service_healthy
      mongo:
        condition: service_healthy
    restart: unless-stopped

# --8<-- [start:dev_volumes]
volumes:
  mongo_data:
  redis_data:
  grafana_data:
  victoria_metrics_data:
  shared_ca:
  loki_data:
  kafka_data:
  kafka_logs:
# --8<-- [end:dev_volumes]

networks:
  app-network:
    driver: bridge