From 3d736ea407bc67ac4b368f78d54363465063598a Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 30 Jan 2026 20:03:58 +0100 Subject: [PATCH 1/5] using toml lib+configs instead of dotenv stuff --- .github/workflows/stack-tests.yml | 4 +- backend/.env | 84 ------------------- backend/.env.test | 82 ------------------ backend/Dockerfile | 11 ++- backend/Dockerfile.base | 1 + backend/app/main.py | 2 +- backend/app/settings.py | 43 ++++++---- backend/config.coordinator.toml | 2 + backend/config.dlq-processor.toml | 2 + backend/config.event-replay.toml | 2 + backend/config.k8s-worker.toml | 2 + backend/config.pod-monitor.toml | 2 + backend/config.result-processor.toml | 2 + backend/config.saga-orchestrator.toml | 2 + backend/config.test.toml | 79 +++++++++++++++++ backend/config.toml | 78 +++++++++++++++++ backend/pyproject.toml | 1 - backend/scripts/create_topics.py | 2 +- backend/tests/conftest.py | 2 +- backend/tests/load/config.py | 17 ++-- .../unit/services/pod_monitor/test_monitor.py | 4 +- backend/uv.lock | 24 ------ backend/workers/dlq_processor.py | 2 +- backend/workers/run_coordinator.py | 2 +- backend/workers/run_event_replay.py | 2 +- backend/workers/run_k8s_worker.py | 2 +- backend/workers/run_pod_monitor.py | 2 +- backend/workers/run_result_processor.py | 2 +- backend/workers/run_saga_orchestrator.py | 2 +- deploy.sh | 4 +- docker-compose.yaml | 60 ++++--------- 31 files changed, 241 insertions(+), 285 deletions(-) delete mode 100644 backend/.env delete mode 100644 backend/.env.test create mode 100644 backend/config.coordinator.toml create mode 100644 backend/config.dlq-processor.toml create mode 100644 backend/config.event-replay.toml create mode 100644 backend/config.k8s-worker.toml create mode 100644 backend/config.pod-monitor.toml create mode 100644 backend/config.result-processor.toml create mode 100644 backend/config.saga-orchestrator.toml create mode 100644 backend/config.test.toml create mode 100644 backend/config.toml diff --git a/.github/workflows/stack-tests.yml b/.github/workflows/stack-tests.yml index 390fba13..c63d4c8d 100644 --- a/.github/workflows/stack-tests.yml +++ b/.github/workflows/stack-tests.yml @@ -211,7 +211,7 @@ jobs: uses: ./.github/actions/k3s-setup - name: Use test environment config - run: cp backend/.env.test backend/.env + run: cp backend/config.test.toml backend/config.toml - name: Start stack run: ./deploy.sh dev --wait @@ -322,7 +322,7 @@ jobs: uses: ./.github/actions/k3s-setup - name: Use test environment config - run: cp backend/.env.test backend/.env + run: cp backend/config.test.toml backend/config.toml - name: Start stack run: ./deploy.sh dev --wait diff --git a/backend/.env b/backend/.env deleted file mode 100644 index fb5f814b..00000000 --- a/backend/.env +++ /dev/null @@ -1,84 +0,0 @@ -PROJECT_NAME=integr8scode -DATABASE_NAME=integr8scode_db -SECRET_KEY=${SECRET_KEY:-uS5xBF-OKXHV-1vqU4ASLwyPcKpSdUTLqGHPYs3y-Yc} -ALGORITHM=HS256 -ACCESS_TOKEN_EXPIRE_MINUTES=1440 -MONGO_ROOT_USER="${MONGO_ROOT_USER:-root}" -MONGO_ROOT_PASSWORD="${MONGO_ROOT_PASSWORD:-rootpassword}" -MONGODB_URL="mongodb://${MONGO_ROOT_USER}:${MONGO_ROOT_PASSWORD}@mongo:27017/integr8scode?authSource=admin" -KUBERNETES_CONFIG_PATH=/app/kubeconfig.yaml -KUBERNETES_CA_CERTIFICATE_PATH=/app/certs/k8s-ca.pem -K8S_POD_CPU_LIMIT=1000m -K8S_POD_MEMORY_LIMIT=128Mi -K8S_POD_CPU_REQUEST=200m -K8S_POD_MEMORY_REQUEST=128Mi -K8S_POD_EXECUTION_TIMEOUT=5 -K8S_NAMESPACE=integr8scode -RATE_LIMITS=100/minute - -# Event-Driven Design Configuration -KAFKA_BOOTSTRAP_SERVERS=kafka:29092 -SCHEMA_REGISTRY_URL=http://schema-registry:8081 -ENABLE_EVENT_STREAMING=true -EVENT_RETENTION_DAYS=30 -KAFKA_CONSUMER_GROUP_ID=integr8scode-backend -KAFKA_AUTO_OFFSET_RESET=earliest -KAFKA_ENABLE_AUTO_COMMIT=true -KAFKA_SESSION_TIMEOUT_MS=45000 -KAFKA_HEARTBEAT_INTERVAL_MS=10000 -KAFKA_REQUEST_TIMEOUT_MS=40000 -KAFKA_MAX_POLL_RECORDS=500 - -# SSE Configuration -SSE_CONSUMER_POOL_SIZE=10 -SSE_HEARTBEAT_INTERVAL=30 - -# Logging Configuration -LOG_LEVEL=DEBUG - -# Distributed Tracing -ENABLE_TRACING=true -JAEGER_AGENT_HOST=jaeger -JAEGER_AGENT_PORT=6831 -TRACING_SERVICE_NAME=integr8scode-backend -TRACING_SERVICE_VERSION=1.0.0 -TRACING_SAMPLING_RATE=1.0 - -# Dead Letter Queue Configuration -DLQ_RETRY_MAX_ATTEMPTS=5 -DLQ_RETRY_BASE_DELAY_SECONDS=60.0 -DLQ_RETRY_MAX_DELAY_SECONDS=3600.0 -DLQ_RETENTION_DAYS=7 -DLQ_WARNING_THRESHOLD=100 -DLQ_CRITICAL_THRESHOLD=1000 - -# App URL for notification links -APP_URL=https://localhost - -# Service Configuration -SERVICE_NAME=integr8scode-backend -SERVICE_VERSION=1.0.0 - -# OpenTelemetry Configuration -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 -OTEL_SERVICE_NAME=integr8scode-backend -OTEL_SERVICE_VERSION=1.0.0 -OTEL_RESOURCE_ATTRIBUTES=environment=production,team=backend - -# Web server concurrency settings (Gunicorn + Uvicorn workers) -# Tune these for your machine. Defaults are safe for dev. -WEB_CONCURRENCY=4 -WEB_THREADS=4 -WEB_TIMEOUT=60 -WEB_BACKLOG=2048 - -# Local development server bind address -# When running uvicorn locally (outside Docker), bind to IPv4 loopback to avoid -# IPv6-only localhost resolution on some Linux distros. -SERVER_HOST=127.0.0.1 - -# Security -BCRYPT_ROUNDS=12 - -# Redis Configuration -REDIS_MAX_CONNECTIONS=200 diff --git a/backend/.env.test b/backend/.env.test deleted file mode 100644 index e0e24652..00000000 --- a/backend/.env.test +++ /dev/null @@ -1,82 +0,0 @@ -PROJECT_NAME=integr8scode -DATABASE_NAME=integr8scode_db -SECRET_KEY=${SECRET_KEY:-uS5xBF-OKXHV-1vqU4ASLwyPcKpSdUTLqGHPYs3y-Yc} -ALGORITHM=HS256 -ACCESS_TOKEN_EXPIRE_MINUTES=1440 -MONGO_ROOT_USER="${MONGO_ROOT_USER:-root}" -MONGO_ROOT_PASSWORD="${MONGO_ROOT_PASSWORD:-rootpassword}" -MONGODB_URL="mongodb://${MONGO_ROOT_USER}:${MONGO_ROOT_PASSWORD}@mongo:27017/integr8scode?authSource=admin" -KUBERNETES_CONFIG_PATH=/app/kubeconfig.yaml -KUBERNETES_CA_CERTIFICATE_PATH=/app/certs/k8s-ca.pem -K8S_POD_CPU_LIMIT=1000m -K8S_POD_MEMORY_LIMIT=128Mi -K8S_POD_CPU_REQUEST=50m -K8S_POD_MEMORY_REQUEST=128Mi -K8S_POD_EXECUTION_TIMEOUT=5 -K8S_NAMESPACE=integr8scode -RATE_LIMITS=99999/second -RATE_LIMIT_ENABLED=false - -# Event-Driven Design Configuration -KAFKA_BOOTSTRAP_SERVERS=kafka:29092 -SCHEMA_REGISTRY_URL=http://schema-registry:8081 -ENABLE_EVENT_STREAMING=true -EVENT_RETENTION_DAYS=30 -KAFKA_CONSUMER_GROUP_ID=integr8scode-backend -KAFKA_AUTO_OFFSET_RESET=earliest -KAFKA_ENABLE_AUTO_COMMIT=true -KAFKA_SESSION_TIMEOUT_MS=10000 -KAFKA_HEARTBEAT_INTERVAL_MS=3000 -KAFKA_REQUEST_TIMEOUT_MS=15000 -KAFKA_MAX_POLL_RECORDS=500 - -# SSE Configuration -SSE_CONSUMER_POOL_SIZE=10 -SSE_HEARTBEAT_INTERVAL=30 - -# Logging Configuration -LOG_LEVEL=WARNING - -# Distributed Tracing -ENABLE_TRACING=true -JAEGER_AGENT_HOST=jaeger -JAEGER_AGENT_PORT=6831 -TRACING_SERVICE_NAME=integr8scode-backend -TRACING_SERVICE_VERSION=1.0.0 -TRACING_SAMPLING_RATE=1.0 - -# Dead Letter Queue Configuration -DLQ_RETRY_MAX_ATTEMPTS=5 -DLQ_RETRY_BASE_DELAY_SECONDS=60.0 -DLQ_RETRY_MAX_DELAY_SECONDS=3600.0 -DLQ_RETENTION_DAYS=7 -DLQ_WARNING_THRESHOLD=100 -DLQ_CRITICAL_THRESHOLD=1000 - -# App URL for notification links -APP_URL=https://localhost - -# Service Configuration -SERVICE_NAME=integr8scode-backend -SERVICE_VERSION=1.0.0 - -# OpenTelemetry Configuration -OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 -OTEL_SERVICE_NAME=integr8scode-backend -OTEL_SERVICE_VERSION=1.0.0 -OTEL_RESOURCE_ATTRIBUTES=environment=test,team=backend - -# Web server concurrency settings (Gunicorn + Uvicorn workers) -# Tune these for your machine. Defaults are safe for dev. -WEB_CONCURRENCY=1 -WEB_THREADS=4 -WEB_TIMEOUT=60 -WEB_BACKLOG=2048 - -# Local development server bind address -# When running uvicorn locally (outside Docker), bind to IPv4 loopback to avoid -# IPv6-only localhost resolution on some Linux distros. -SERVER_HOST=127.0.0.1 - -# Security -BCRYPT_ROUNDS=4 diff --git a/backend/Dockerfile b/backend/Dockerfile index 97ab3a74..d0c04500 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -5,7 +5,7 @@ FROM base COPY ./app /app/app COPY ./workers /app/workers COPY ./scripts /app/scripts -COPY .env /app/.env +COPY config.toml /app/config.toml COPY openssl.cnf /app/openssl.cnf # Ensure the certs directory exists @@ -25,17 +25,16 @@ while [ ! -f /app/certs/server.key ]; do done echo "Starting application..." -[ -f /app/kubeconfig.yaml ] && export KUBECONFIG=/app/kubeconfig.yaml exec gunicorn 'app.main:create_app()' \ -k uvicorn.workers.UvicornWorker \ --bind 0.0.0.0:443 \ - --workers ${WEB_CONCURRENCY:-4} \ - --threads ${WEB_THREADS:-1} \ - --timeout ${WEB_TIMEOUT:-60} \ + --workers 4 \ + --threads 4 \ + --timeout 60 \ --graceful-timeout 30 \ --keep-alive 2 \ - --backlog ${WEB_BACKLOG:-2048} \ + --backlog 2048 \ --log-level info \ --access-logfile - \ --error-logfile - \ diff --git a/backend/Dockerfile.base b/backend/Dockerfile.base index c06680c4..b9678c4f 100644 --- a/backend/Dockerfile.base +++ b/backend/Dockerfile.base @@ -24,3 +24,4 @@ RUN uv sync --locked --no-dev --no-install-project # Set paths: PYTHONPATH for imports, PATH for venv binaries (no uv run needed at runtime) ENV PYTHONPATH=/app ENV PATH="/app/.venv/bin:$PATH" +ENV KUBECONFIG=/app/kubeconfig.yaml diff --git a/backend/app/main.py b/backend/app/main.py index bf776a41..5dfc1e7b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -48,7 +48,7 @@ def create_app(settings: Settings | None = None) -> FastAPI: Args: settings: Optional pre-configured settings (e.g., TestSettings for testing). - If None, creates Settings() which reads from env vars then .env file. + If None, loads from config.toml. """ settings = settings or Settings() logger = setup_logger(settings.LOG_LEVEL) diff --git a/backend/app/settings.py b/backend/app/settings.py index 2bcf2bd6..3851fbff 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -1,19 +1,39 @@ +import tomllib from typing import Literal -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import BaseModel, ConfigDict, Field from app.domain.execution import LanguageInfoDomain from app.runtime_registry import EXAMPLE_SCRIPTS as EXEC_EXAMPLE_SCRIPTS from app.runtime_registry import SUPPORTED_RUNTIMES as RUNTIME_MATRIX -class Settings(BaseSettings): +class Settings(BaseModel): + """Application settings loaded from TOML configuration files. + + All config is read from TOML — no environment variables, no .env files. + + Usage: + Settings() # loads config.toml + Settings(config_path="config.test.toml") # loads test config + Settings(override_path="config.coordinator.toml") # base + per-worker override + """ + + model_config = ConfigDict(extra="forbid") + + def __init__(self, config_path: str = "config.toml", override_path: str | None = None) -> None: + with open(config_path, "rb") as f: + data = tomllib.load(f) + if override_path: + with open(override_path, "rb") as f: + data |= tomllib.load(f) + super().__init__(**data) + PROJECT_NAME: str = "integr8scode" DATABASE_NAME: str = "integr8scode_db" API_V1_STR: str = "/api/v1" SECRET_KEY: str = Field( - ..., # Actual key be loaded from .env file + ..., min_length=32, description="Secret key for JWT token signing. Must be at least 32 characters.", ) @@ -145,27 +165,14 @@ class Settings(BaseSettings): OTEL_RESOURCE_ATTRIBUTES: str | None = None # Web server (Gunicorn/Uvicorn) concurrency settings - # These are read from environment and used by the runtime entrypoint - # and by app.main when started directly via uvicorn. WEB_CONCURRENCY: int = 4 WEB_THREADS: int = 1 WEB_TIMEOUT: int = 60 WEB_BACKLOG: int = 2048 - # Additional MongoDB settings (for docker-compose compatibility) - MONGO_ROOT_USER: str | None = None - MONGO_ROOT_PASSWORD: str | None = None - # Development mode detection DEVELOPMENT_MODE: bool = False - SECURE_COOKIES: bool = True # Can be overridden in .env for development + SECURE_COOKIES: bool = True # Logging configuration LOG_LEVEL: str = Field(default="DEBUG", description="Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)") - - model_config = SettingsConfigDict( - env_file=".env", - env_file_encoding="utf-8", - case_sensitive=True, - extra="forbid", # Raise error on extra fields - ) diff --git a/backend/config.coordinator.toml b/backend/config.coordinator.toml new file mode 100644 index 00000000..067768bb --- /dev/null +++ b/backend/config.coordinator.toml @@ -0,0 +1,2 @@ +TRACING_SERVICE_NAME = "execution-coordinator" +KAFKA_CONSUMER_GROUP_ID = "execution-coordinator" diff --git a/backend/config.dlq-processor.toml b/backend/config.dlq-processor.toml new file mode 100644 index 00000000..5d8447fc --- /dev/null +++ b/backend/config.dlq-processor.toml @@ -0,0 +1,2 @@ +TRACING_SERVICE_NAME = "dlq-processor" +KAFKA_CONSUMER_GROUP_ID = "dlq-processor" diff --git a/backend/config.event-replay.toml b/backend/config.event-replay.toml new file mode 100644 index 00000000..92ea9e78 --- /dev/null +++ b/backend/config.event-replay.toml @@ -0,0 +1,2 @@ +TRACING_SERVICE_NAME = "event-replay" +KAFKA_CONSUMER_GROUP_ID = "event-replay" diff --git a/backend/config.k8s-worker.toml b/backend/config.k8s-worker.toml new file mode 100644 index 00000000..2fb75283 --- /dev/null +++ b/backend/config.k8s-worker.toml @@ -0,0 +1,2 @@ +TRACING_SERVICE_NAME = "k8s-worker" +KAFKA_CONSUMER_GROUP_ID = "k8s-worker" diff --git a/backend/config.pod-monitor.toml b/backend/config.pod-monitor.toml new file mode 100644 index 00000000..2f2a45e6 --- /dev/null +++ b/backend/config.pod-monitor.toml @@ -0,0 +1,2 @@ +TRACING_SERVICE_NAME = "pod-monitor" +KAFKA_CONSUMER_GROUP_ID = "pod-monitor" diff --git a/backend/config.result-processor.toml b/backend/config.result-processor.toml new file mode 100644 index 00000000..ae697d60 --- /dev/null +++ b/backend/config.result-processor.toml @@ -0,0 +1,2 @@ +TRACING_SERVICE_NAME = "result-processor" +KAFKA_CONSUMER_GROUP_ID = "result-processor" diff --git a/backend/config.saga-orchestrator.toml b/backend/config.saga-orchestrator.toml new file mode 100644 index 00000000..6d259608 --- /dev/null +++ b/backend/config.saga-orchestrator.toml @@ -0,0 +1,2 @@ +TRACING_SERVICE_NAME = "saga-orchestrator" +KAFKA_CONSUMER_GROUP_ID = "saga-orchestrator" diff --git a/backend/config.test.toml b/backend/config.test.toml new file mode 100644 index 00000000..dff5e9c7 --- /dev/null +++ b/backend/config.test.toml @@ -0,0 +1,79 @@ +# Integr8sCode backend test configuration +# Differences from config.toml: lower timeouts, faster bcrypt, relaxed rate limits + +PROJECT_NAME = "integr8scode" +DATABASE_NAME = "integr8scode_db" +SECRET_KEY = "uS5xBF-OKXHV-1vqU4ASLwyPcKpSdUTLqGHPYs3y-Yc" +ALGORITHM = "HS256" +ACCESS_TOKEN_EXPIRE_MINUTES = 1440 + +MONGODB_URL = "mongodb://root:rootpassword@mongo:27017/integr8scode?authSource=admin" + +KUBERNETES_CONFIG_PATH = "/app/kubeconfig.yaml" +KUBERNETES_CA_CERTIFICATE_PATH = "/app/certs/k8s-ca.pem" +K8S_POD_CPU_LIMIT = "1000m" +K8S_POD_MEMORY_LIMIT = "128Mi" +K8S_POD_CPU_REQUEST = "50m" +K8S_POD_MEMORY_REQUEST = "128Mi" +K8S_POD_EXECUTION_TIMEOUT = 5 +K8S_NAMESPACE = "integr8scode" + +RATE_LIMITS = "99999/second" +RATE_LIMIT_ENABLED = false + +SERVER_HOST = "0.0.0.0" + +# Kafka (lower timeouts for tests) +KAFKA_BOOTSTRAP_SERVERS = "kafka:29092" +SCHEMA_REGISTRY_URL = "http://schema-registry:8081" +ENABLE_EVENT_STREAMING = true +EVENT_RETENTION_DAYS = 30 +KAFKA_CONSUMER_GROUP_ID = "integr8scode-backend" +KAFKA_AUTO_OFFSET_RESET = "earliest" +KAFKA_ENABLE_AUTO_COMMIT = true +KAFKA_SESSION_TIMEOUT_MS = 10000 +KAFKA_HEARTBEAT_INTERVAL_MS = 3000 +KAFKA_REQUEST_TIMEOUT_MS = 15000 +KAFKA_MAX_POLL_RECORDS = 500 + +# SSE +SSE_CONSUMER_POOL_SIZE = 10 +SSE_HEARTBEAT_INTERVAL = 30 + +LOG_LEVEL = "WARNING" + +# Tracing +ENABLE_TRACING = true +JAEGER_AGENT_HOST = "jaeger" +JAEGER_AGENT_PORT = 6831 +TRACING_SERVICE_NAME = "integr8scode-backend" +TRACING_SERVICE_VERSION = "1.0.0" +TRACING_SAMPLING_RATE = 1.0 + +# DLQ +DLQ_RETRY_MAX_ATTEMPTS = 5 +DLQ_RETRY_BASE_DELAY_SECONDS = 60.0 +DLQ_RETRY_MAX_DELAY_SECONDS = 3600.0 +DLQ_RETENTION_DAYS = 7 +DLQ_WARNING_THRESHOLD = 100 +DLQ_CRITICAL_THRESHOLD = 1000 + +APP_URL = "https://localhost" + +SERVICE_NAME = "integr8scode-backend" +SERVICE_VERSION = "1.0.0" + +# OpenTelemetry +OTEL_EXPORTER_OTLP_ENDPOINT = "http://otel-collector:4317" +OTEL_SERVICE_NAME = "integr8scode-backend" +OTEL_SERVICE_VERSION = "1.0.0" +OTEL_RESOURCE_ATTRIBUTES = "environment=test,team=backend" + +# Gunicorn / Uvicorn +WEB_CONCURRENCY = 1 +WEB_THREADS = 4 +WEB_TIMEOUT = 60 +WEB_BACKLOG = 2048 + +BCRYPT_ROUNDS = 4 +REDIS_MAX_CONNECTIONS = 200 diff --git a/backend/config.toml b/backend/config.toml new file mode 100644 index 00000000..a449408b --- /dev/null +++ b/backend/config.toml @@ -0,0 +1,78 @@ +# Integr8sCode backend configuration +# All settings loaded via tomllib + pydantic BaseModel.model_validate() + +PROJECT_NAME = "integr8scode" +DATABASE_NAME = "integr8scode_db" +SECRET_KEY = "uS5xBF-OKXHV-1vqU4ASLwyPcKpSdUTLqGHPYs3y-Yc" +ALGORITHM = "HS256" +ACCESS_TOKEN_EXPIRE_MINUTES = 1440 + +MONGODB_URL = "mongodb://root:rootpassword@mongo:27017/integr8scode?authSource=admin" + +KUBERNETES_CONFIG_PATH = "/app/kubeconfig.yaml" +KUBERNETES_CA_CERTIFICATE_PATH = "/app/certs/k8s-ca.pem" +K8S_POD_CPU_LIMIT = "1000m" +K8S_POD_MEMORY_LIMIT = "128Mi" +K8S_POD_CPU_REQUEST = "200m" +K8S_POD_MEMORY_REQUEST = "128Mi" +K8S_POD_EXECUTION_TIMEOUT = 5 +K8S_NAMESPACE = "integr8scode" + +RATE_LIMITS = "100/minute" + +SERVER_HOST = "0.0.0.0" + +# Kafka +KAFKA_BOOTSTRAP_SERVERS = "kafka:29092" +SCHEMA_REGISTRY_URL = "http://schema-registry:8081" +ENABLE_EVENT_STREAMING = true +EVENT_RETENTION_DAYS = 30 +KAFKA_CONSUMER_GROUP_ID = "integr8scode-backend" +KAFKA_AUTO_OFFSET_RESET = "earliest" +KAFKA_ENABLE_AUTO_COMMIT = true +KAFKA_SESSION_TIMEOUT_MS = 45000 +KAFKA_HEARTBEAT_INTERVAL_MS = 10000 +KAFKA_REQUEST_TIMEOUT_MS = 40000 +KAFKA_MAX_POLL_RECORDS = 500 + +# SSE +SSE_CONSUMER_POOL_SIZE = 10 +SSE_HEARTBEAT_INTERVAL = 30 + +LOG_LEVEL = "DEBUG" + +# Tracing +ENABLE_TRACING = true +JAEGER_AGENT_HOST = "jaeger" +JAEGER_AGENT_PORT = 6831 +TRACING_SERVICE_NAME = "integr8scode-backend" +TRACING_SERVICE_VERSION = "1.0.0" +TRACING_SAMPLING_RATE = 1.0 + +# DLQ +DLQ_RETRY_MAX_ATTEMPTS = 5 +DLQ_RETRY_BASE_DELAY_SECONDS = 60.0 +DLQ_RETRY_MAX_DELAY_SECONDS = 3600.0 +DLQ_RETENTION_DAYS = 7 +DLQ_WARNING_THRESHOLD = 100 +DLQ_CRITICAL_THRESHOLD = 1000 + +APP_URL = "https://localhost" + +SERVICE_NAME = "integr8scode-backend" +SERVICE_VERSION = "1.0.0" + +# OpenTelemetry +OTEL_EXPORTER_OTLP_ENDPOINT = "http://otel-collector:4317" +OTEL_SERVICE_NAME = "integr8scode-backend" +OTEL_SERVICE_VERSION = "1.0.0" +OTEL_RESOURCE_ATTRIBUTES = "environment=production,team=backend" + +# Gunicorn / Uvicorn +WEB_CONCURRENCY = 4 +WEB_THREADS = 4 +WEB_TIMEOUT = 60 +WEB_BACKLOG = 2048 + +BCRYPT_ROUNDS = 12 +REDIS_MAX_CONNECTIONS = 200 diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1c970094..40fb8763 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -90,7 +90,6 @@ dependencies = [ "pyasn1_modules==0.4.2", "pydantic==2.9.2", "pydantic-avro==0.9.1", - "pydantic-settings==2.5.2", "pydantic_core==2.23.4", "Pygments==2.19.2", "PyJWT==2.9.0", diff --git a/backend/scripts/create_topics.py b/backend/scripts/create_topics.py index c7695a0c..2cf81e21 100755 --- a/backend/scripts/create_topics.py +++ b/backend/scripts/create_topics.py @@ -96,7 +96,7 @@ async def create_topics(settings: Settings) -> None: async def main() -> None: - """Main entry point - creates Settings() which reads from env vars then .env file.""" + """Main entry point - loads settings from config.toml.""" logger.info("Starting Kafka topic creation...") try: diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index ce149796..8062aab3 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -31,7 +31,7 @@ def test_settings() -> Settings: - Kafka: Tests with consumers use xdist_group markers for serial execution - Redis: Per-worker DB number (0-15) to avoid key collisions """ - base = Settings(_env_file=".env.test") + base = Settings(config_path="config.test.toml") return base.model_copy(update={"REDIS_DB": _get_worker_num() % 16}) diff --git a/backend/tests/load/config.py b/backend/tests/load/config.py index b82189fe..834affbd 100644 --- a/backend/tests/load/config.py +++ b/backend/tests/load/config.py @@ -2,15 +2,12 @@ from typing import Literal -from pydantic import Field -from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import BaseModel Mode = Literal["monkey", "user", "both"] -class LoadConfig(BaseSettings): - model_config = SettingsConfigDict(env_prefix="LOAD_", case_sensitive=False) - +class LoadConfig(BaseModel): base_url: str = "https://[::1]:443" api_prefix: str = "/api/v1" verify_tls: bool = False @@ -21,19 +18,19 @@ class LoadConfig(BaseSettings): clients: int = 25 concurrency: int = 10 # Default run duration ~3 minutes - duration_seconds: int = Field(default=180, validation_alias="LOAD_DURATION") - ramp_up_seconds: int = Field(default=5, validation_alias="LOAD_RAMP") + duration_seconds: int = 180 + ramp_up_seconds: int = 5 # User pool (for user-mode) - auto_register_users: bool = Field(default=True, validation_alias="LOAD_AUTO_REGISTER") + auto_register_users: bool = True user_prefix: str = "loaduser" user_domain: str = "example.com" user_password: str = "testpass123!" # Endpoint toggles enable_sse: bool = True - enable_saved_scripts: bool = Field(default=True, validation_alias="LOAD_ENABLE_SCRIPTS") - enable_user_settings: bool = Field(default=True, validation_alias="LOAD_ENABLE_SETTINGS") + enable_saved_scripts: bool = True + enable_user_settings: bool = True enable_notifications: bool = True # Reporting diff --git a/backend/tests/unit/services/pod_monitor/test_monitor.py b/backend/tests/unit/services/pod_monitor/test_monitor.py index af84e50b..a3b89227 100644 --- a/backend/tests/unit/services/pod_monitor/test_monitor.py +++ b/backend/tests/unit/services/pod_monitor/test_monitor.py @@ -80,7 +80,7 @@ def create_test_kafka_event_service(event_metrics: EventMetrics) -> tuple[KafkaE """Create real KafkaEventService with fake dependencies for testing.""" fake_producer = FakeUnifiedProducer() fake_repo = FakeEventRepository() - settings = Settings() # Uses defaults/env vars + settings = Settings(config_path="config.test.toml") service = KafkaEventService( event_repository=fake_repo, @@ -428,7 +428,7 @@ async def produce( failing_service = KafkaEventService( event_repository=fake_repo, kafka_producer=failing_producer, - settings=Settings(), + settings=Settings(config_path="config.test.toml"), logger=_test_logger, event_metrics=event_metrics, ) diff --git a/backend/uv.lock b/backend/uv.lock index 2c500df1..20246a48 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -1122,7 +1122,6 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-avro" }, { name = "pydantic-core" }, - { name = "pydantic-settings" }, { name = "pygments" }, { name = "pyjwt" }, { name = "pymongo" }, @@ -1267,7 +1266,6 @@ requires-dist = [ { name = "pydantic", specifier = "==2.9.2" }, { name = "pydantic-avro", specifier = "==0.9.1" }, { name = "pydantic-core", specifier = "==2.23.4" }, - { name = "pydantic-settings", specifier = "==2.5.2" }, { name = "pygments", specifier = "==2.19.2" }, { name = "pyjwt", specifier = "==2.9.0" }, { name = "pymongo", specifier = "==4.12.1" }, @@ -2481,19 +2479,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/ae/e14b0ff8b3f48e02394d8acd911376b7b66e164535687ef7dc24ea03072f/pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5", size = 1919411, upload-time = "2024-09-16T16:05:18.934Z" }, ] -[[package]] -name = "pydantic-settings" -version = "2.5.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/68/27/0bed9dd26b93328b60a1402febc780e7be72b42847fa8b5c94b7d0aeb6d1/pydantic_settings-2.5.2.tar.gz", hash = "sha256:f90b139682bee4d2065273d5185d71d37ea46cfe57e1b5ae184fc6a0b2484ca0", size = 70938, upload-time = "2024-09-11T09:08:08.489Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/8d/29e82e333f32d9e2051c10764b906c2a6cd140992910b5f49762790911ba/pydantic_settings-2.5.2-py3-none-any.whl", hash = "sha256:2c912e55fd5794a59bf8c832b9de832dcfdf4778d79ff79b708744eed499a907", size = 26864, upload-time = "2024-09-11T09:08:07.242Z" }, -] - [[package]] name = "pygments" version = "2.19.2" @@ -2647,15 +2632,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] -[[package]] -name = "python-dotenv" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/bc/57/e84d88dfe0aec03b7a2d4327012c1627ab5f03652216c63d49846d7a6c58/python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca", size = 39115, upload-time = "2024-01-23T06:33:00.505Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", size = 19863, upload-time = "2024-01-23T06:32:58.246Z" }, -] - [[package]] name = "python-json-logger" version = "2.0.7" diff --git a/backend/workers/dlq_processor.py b/backend/workers/dlq_processor.py index 97598539..febcebc7 100644 --- a/backend/workers/dlq_processor.py +++ b/backend/workers/dlq_processor.py @@ -108,4 +108,4 @@ def signal_handler() -> None: if __name__ == "__main__": - asyncio.run(main(Settings())) + asyncio.run(main(Settings(override_path="config.dlq-processor.toml"))) diff --git a/backend/workers/run_coordinator.py b/backend/workers/run_coordinator.py index 4ddd8984..03702b49 100644 --- a/backend/workers/run_coordinator.py +++ b/backend/workers/run_coordinator.py @@ -47,7 +47,7 @@ async def run_coordinator(settings: Settings) -> None: def main() -> None: """Main entry point for coordinator worker""" - settings = Settings() + settings = Settings(override_path="config.coordinator.toml") logger = setup_logger(settings.LOG_LEVEL) logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") diff --git a/backend/workers/run_event_replay.py b/backend/workers/run_event_replay.py index 95c38dad..fed2deb3 100644 --- a/backend/workers/run_event_replay.py +++ b/backend/workers/run_event_replay.py @@ -59,7 +59,7 @@ async def _cancel_task() -> None: def main() -> None: """Main entry point for event replay service""" - settings = Settings() + settings = Settings(override_path="config.event-replay.toml") logger = setup_logger(settings.LOG_LEVEL) logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") diff --git a/backend/workers/run_k8s_worker.py b/backend/workers/run_k8s_worker.py index ddba5c46..6fe841c1 100644 --- a/backend/workers/run_k8s_worker.py +++ b/backend/workers/run_k8s_worker.py @@ -60,7 +60,7 @@ async def run_kubernetes_worker(settings: Settings) -> None: def main() -> None: """Main entry point for Kubernetes worker""" - settings = Settings() + settings = Settings(override_path="config.k8s-worker.toml") logger = setup_logger(settings.LOG_LEVEL) logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") diff --git a/backend/workers/run_pod_monitor.py b/backend/workers/run_pod_monitor.py index 4b4dd325..0fbee75b 100644 --- a/backend/workers/run_pod_monitor.py +++ b/backend/workers/run_pod_monitor.py @@ -54,7 +54,7 @@ async def run_pod_monitor(settings: Settings) -> None: def main() -> None: """Main entry point for pod monitor worker""" - settings = Settings() + settings = Settings(override_path="config.pod-monitor.toml") logger = setup_logger(settings.LOG_LEVEL) logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") diff --git a/backend/workers/run_result_processor.py b/backend/workers/run_result_processor.py index 1576c8e0..e51d3cfe 100644 --- a/backend/workers/run_result_processor.py +++ b/backend/workers/run_result_processor.py @@ -82,7 +82,7 @@ async def run_result_processor(settings: Settings) -> None: def main() -> None: """Main entry point for result processor worker""" - settings = Settings() + settings = Settings(override_path="config.result-processor.toml") logger = setup_logger(settings.LOG_LEVEL) logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") diff --git a/backend/workers/run_saga_orchestrator.py b/backend/workers/run_saga_orchestrator.py index 7fd0c359..f54ccfa1 100644 --- a/backend/workers/run_saga_orchestrator.py +++ b/backend/workers/run_saga_orchestrator.py @@ -52,7 +52,7 @@ async def run_saga_orchestrator(settings: Settings) -> None: def main() -> None: """Main entry point for saga orchestrator worker""" - settings = Settings() + settings = Settings(override_path="config.saga-orchestrator.toml") logger = setup_logger(settings.LOG_LEVEL) logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s") diff --git a/deploy.sh b/deploy.sh index 66f6cf8f..f25c480f 100755 --- a/deploy.sh +++ b/deploy.sh @@ -79,8 +79,8 @@ show_help() { echo " --set key=value Override Helm values" echo "" echo "Configuration:" - echo " All settings come from backend/.env (single source of truth)" - echo " For CI/tests: cp backend/.env.test backend/.env" + echo " All settings come from backend/config.toml (single source of truth)" + echo " For CI/tests: cp backend/config.test.toml backend/config.toml" echo "" echo "Examples:" echo " ./deploy.sh dev # Start dev environment" diff --git a/docker-compose.yaml b/docker-compose.yaml index ed5a3379..7ce0aa89 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -106,7 +106,8 @@ services: - ./backend/scripts:/app/scripts - ./backend/tests:/app/tests:ro - ./backend/certs:/app/certs:ro - - ./backend/.env.test:/app/.env.test:ro + - ./backend/config.test.toml:/app/config.test.toml:ro + - ./backend/config.toml:/app/config.toml:ro - shared_ca:/shared_ca:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro ports: @@ -117,11 +118,6 @@ services: container_name: backend extra_hosts: - "host.docker.internal:host-gateway" - env_file: - - ./backend/.env - environment: - - SERVER_HOST=0.0.0.0 - - KUBECONFIG=/app/kubeconfig.yaml healthcheck: # Simpler, reliable healthcheck: curl fails non-zero for HTTP >=400 with -f test: ["CMD-SHELL", "curl -k -f -s https://localhost:443/api/v1/health/live >/dev/null || exit 1"] @@ -423,14 +419,11 @@ services: condition: service_completed_successfully mongo: condition: service_started - env_file: - - ./backend/.env - environment: - - TRACING_SERVICE_NAME=execution-coordinator - - KAFKA_CONSUMER_GROUP_ID=execution-coordinator volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro + - ./backend/config.toml:/app/config.toml:ro + - ./backend/config.coordinator.toml:/app/config.coordinator.toml:ro networks: - app-network restart: unless-stopped @@ -449,16 +442,12 @@ services: condition: service_completed_successfully mongo: condition: service_started - env_file: - - ./backend/.env - environment: - - TRACING_SERVICE_NAME=k8s-worker - - KAFKA_CONSUMER_GROUP_ID=k8s-worker - - KUBECONFIG=/app/kubeconfig.yaml volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro + - ./backend/config.toml:/app/config.toml:ro + - ./backend/config.k8s-worker.toml:/app/config.k8s-worker.toml:ro networks: - app-network extra_hosts: @@ -477,16 +466,12 @@ services: condition: service_completed_successfully kafka-init: condition: service_completed_successfully - env_file: - - ./backend/.env - environment: - - TRACING_SERVICE_NAME=pod-monitor - - KAFKA_CONSUMER_GROUP_ID=pod-monitor - - KUBECONFIG=/app/kubeconfig.yaml volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro + - ./backend/config.toml:/app/config.toml:ro + - ./backend/config.pod-monitor.toml:/app/config.pod-monitor.toml:ro networks: - app-network extra_hosts: @@ -507,16 +492,12 @@ services: condition: service_completed_successfully mongo: condition: service_started - env_file: - - ./backend/.env - environment: - - TRACING_SERVICE_NAME=result-processor - - KAFKA_CONSUMER_GROUP_ID=result-processor-group - - KUBECONFIG=/app/kubeconfig.yaml volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro + - ./backend/config.toml:/app/config.toml:ro + - ./backend/config.result-processor.toml:/app/config.result-processor.toml:ro networks: - app-network extra_hosts: @@ -537,14 +518,11 @@ services: condition: service_completed_successfully mongo: condition: service_started - env_file: - - ./backend/.env - environment: - - TRACING_SERVICE_NAME=saga-orchestrator - - KAFKA_CONSUMER_GROUP_ID=saga-orchestrator volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro + - ./backend/config.toml:/app/config.toml:ro + - ./backend/config.saga-orchestrator.toml:/app/config.saga-orchestrator.toml:ro networks: - app-network restart: unless-stopped @@ -585,14 +563,11 @@ services: mongo: condition: service_started command: ["python", "workers/run_event_replay.py"] - env_file: - - ./backend/.env - environment: - - TRACING_SERVICE_NAME=event-replay - - KAFKA_CONSUMER_GROUP_ID=event-replay volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro + - ./backend/config.toml:/app/config.toml:ro + - ./backend/config.event-replay.toml:/app/config.event-replay.toml:ro networks: - app-network restart: unless-stopped @@ -613,14 +588,11 @@ services: mongo: condition: service_started command: ["python", "workers/dlq_processor.py"] - env_file: - - ./backend/.env - environment: - - TRACING_SERVICE_NAME=dlq-processor - - KAFKA_CONSUMER_GROUP_ID=dlq-processor volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro + - ./backend/config.toml:/app/config.toml:ro + - ./backend/config.dlq-processor.toml:/app/config.dlq-processor.toml:ro networks: - app-network restart: unless-stopped From c49edc361331dbf6ef90ffbef58238abcedc60de Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 30 Jan 2026 20:40:15 +0100 Subject: [PATCH 2/5] feat: secrets.toml is used --- .github/workflows/stack-tests.yml | 8 ++++++-- .gitignore | 3 +++ backend/app/settings.py | 22 ++++++++++++++++++---- backend/config.toml | 9 +++++---- backend/secrets.example.toml | 17 +++++++++++++++++ docker-compose.yaml | 8 ++++++++ 6 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 backend/secrets.example.toml diff --git a/.github/workflows/stack-tests.yml b/.github/workflows/stack-tests.yml index c63d4c8d..15b05d29 100644 --- a/.github/workflows/stack-tests.yml +++ b/.github/workflows/stack-tests.yml @@ -211,7 +211,9 @@ jobs: uses: ./.github/actions/k3s-setup - name: Use test environment config - run: cp backend/config.test.toml backend/config.toml + run: | + cp backend/config.test.toml backend/config.toml + touch backend/secrets.toml - name: Start stack run: ./deploy.sh dev --wait @@ -322,7 +324,9 @@ jobs: uses: ./.github/actions/k3s-setup - name: Use test environment config - run: cp backend/config.test.toml backend/config.toml + run: | + cp backend/config.test.toml backend/config.toml + touch backend/secrets.toml - name: Start stack run: ./deploy.sh dev --wait diff --git a/.gitignore b/.gitignore index ea473114..c49ae4a0 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,9 @@ dmypy.json .env.*.local *.local.env +# TOML secrets (dev values — production mounts from K8s Secret) +secrets.toml + # Certificates (generated locally) certs/ *.pem diff --git a/backend/app/settings.py b/backend/app/settings.py index 3851fbff..d487f6da 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -1,4 +1,5 @@ import tomllib +from pathlib import Path from typing import Literal from pydantic import BaseModel, ConfigDict, Field @@ -13,17 +14,30 @@ class Settings(BaseModel): All config is read from TOML — no environment variables, no .env files. + Load order (each layer overrides the previous): + 1. config_path — base settings (committed to git) + 2. secrets_path — sensitive overrides (gitignored, mounted from K8s Secret in prod) + 3. override_path — per-worker service overrides (TRACING_SERVICE_NAME, etc.) + Usage: - Settings() # loads config.toml - Settings(config_path="config.test.toml") # loads test config - Settings(override_path="config.coordinator.toml") # base + per-worker override + Settings() # config.toml + secrets + Settings(config_path="config.test.toml") # test config (has own secrets) + Settings(override_path="config.coordinator.toml") # base + secrets + worker """ model_config = ConfigDict(extra="forbid") - def __init__(self, config_path: str = "config.toml", override_path: str | None = None) -> None: + def __init__( + self, + config_path: str = "config.toml", + override_path: str | None = None, + secrets_path: str = "secrets.toml", + ) -> None: with open(config_path, "rb") as f: data = tomllib.load(f) + if Path(secrets_path).is_file(): + with open(secrets_path, "rb") as f: + data |= tomllib.load(f) if override_path: with open(override_path, "rb") as f: data |= tomllib.load(f) diff --git a/backend/config.toml b/backend/config.toml index a449408b..c88e20ed 100644 --- a/backend/config.toml +++ b/backend/config.toml @@ -1,13 +1,14 @@ -# Integr8sCode backend configuration -# All settings loaded via tomllib + pydantic BaseModel.model_validate() +# Integr8sCode backend configuration (development defaults). +# Secrets (SECRET_KEY, MONGODB_URL credentials) live in secrets.toml (gitignored). +# Production: mount secrets.toml from a Kubernetes Secret or generate in CI. +# See secrets.example.toml for the required keys. PROJECT_NAME = "integr8scode" DATABASE_NAME = "integr8scode_db" -SECRET_KEY = "uS5xBF-OKXHV-1vqU4ASLwyPcKpSdUTLqGHPYs3y-Yc" ALGORITHM = "HS256" ACCESS_TOKEN_EXPIRE_MINUTES = 1440 -MONGODB_URL = "mongodb://root:rootpassword@mongo:27017/integr8scode?authSource=admin" +MONGODB_URL = "mongodb://mongo:27017/integr8scode" KUBERNETES_CONFIG_PATH = "/app/kubeconfig.yaml" KUBERNETES_CA_CERTIFICATE_PATH = "/app/certs/k8s-ca.pem" diff --git a/backend/secrets.example.toml b/backend/secrets.example.toml new file mode 100644 index 00000000..36b1823b --- /dev/null +++ b/backend/secrets.example.toml @@ -0,0 +1,17 @@ +# Copy to secrets.toml and fill in real values. +# cp secrets.example.toml secrets.toml +# +# Required keys (no defaults in config.toml): +# SECRET_KEY — JWT signing key, min 32 characters +# MONGODB_URL — full connection string with credentials +# +# GitHub Actions: create secrets.toml from repository secrets: +# cat > backend/secrets.toml << EOF +# SECRET_KEY = "${{ secrets.JWT_SECRET_KEY }}" +# MONGODB_URL = "${{ secrets.MONGODB_URL }}" +# EOF +# +# Kubernetes: store as a Secret and mount at /app/secrets.toml + +SECRET_KEY = "CHANGE_ME_min_32_chars_long_!!!!" +MONGODB_URL = "mongodb://user:password@mongo:27017/integr8scode?authSource=admin" diff --git a/docker-compose.yaml b/docker-compose.yaml index 7ce0aa89..51af11d9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -108,6 +108,7 @@ services: - ./backend/certs:/app/certs:ro - ./backend/config.test.toml:/app/config.test.toml:ro - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro - shared_ca:/shared_ca:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro ports: @@ -423,6 +424,7 @@ services: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.coordinator.toml:/app/config.coordinator.toml:ro networks: - app-network @@ -447,6 +449,7 @@ services: - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.k8s-worker.toml:/app/config.k8s-worker.toml:ro networks: - app-network @@ -471,6 +474,7 @@ services: - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.pod-monitor.toml:/app/config.pod-monitor.toml:ro networks: - app-network @@ -497,6 +501,7 @@ services: - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.result-processor.toml:/app/config.result-processor.toml:ro networks: - app-network @@ -522,6 +527,7 @@ services: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.saga-orchestrator.toml:/app/config.saga-orchestrator.toml:ro networks: - app-network @@ -567,6 +573,7 @@ services: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.event-replay.toml:/app/config.event-replay.toml:ro networks: - app-network @@ -592,6 +599,7 @@ services: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.dlq-processor.toml:/app/config.dlq-processor.toml:ro networks: - app-network From 8bda72cff88f2312bd069693edff63a2b58daac8 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 30 Jan 2026 21:11:18 +0100 Subject: [PATCH 3/5] chore: docs update - toml stuff added --- .github/workflows/stack-tests.yml | 4 +- backend/config.test.toml | 4 +- backend/secrets.example.toml | 4 +- docs/getting-started.md | 5 +- docs/index.md | 4 +- docs/operations/deployment.md | 5 +- ...ironment-variables.md => configuration.md} | 133 ++++++++++++------ mkdocs.yml | 2 +- 8 files changed, 107 insertions(+), 54 deletions(-) rename docs/reference/{environment-variables.md => configuration.md} (53%) diff --git a/.github/workflows/stack-tests.yml b/.github/workflows/stack-tests.yml index 15b05d29..b1ef9801 100644 --- a/.github/workflows/stack-tests.yml +++ b/.github/workflows/stack-tests.yml @@ -213,7 +213,7 @@ jobs: - name: Use test environment config run: | cp backend/config.test.toml backend/config.toml - touch backend/secrets.toml + cp backend/secrets.example.toml backend/secrets.toml - name: Start stack run: ./deploy.sh dev --wait @@ -326,7 +326,7 @@ jobs: - name: Use test environment config run: | cp backend/config.test.toml backend/config.toml - touch backend/secrets.toml + cp backend/secrets.example.toml backend/secrets.toml - name: Start stack run: ./deploy.sh dev --wait diff --git a/backend/config.test.toml b/backend/config.test.toml index dff5e9c7..42274458 100644 --- a/backend/config.test.toml +++ b/backend/config.test.toml @@ -1,14 +1,12 @@ # Integr8sCode backend test configuration # Differences from config.toml: lower timeouts, faster bcrypt, relaxed rate limits +# Secrets (SECRET_KEY, MONGODB_URL) live in secrets.toml — use secrets.test.toml in CI. PROJECT_NAME = "integr8scode" DATABASE_NAME = "integr8scode_db" -SECRET_KEY = "uS5xBF-OKXHV-1vqU4ASLwyPcKpSdUTLqGHPYs3y-Yc" ALGORITHM = "HS256" ACCESS_TOKEN_EXPIRE_MINUTES = 1440 -MONGODB_URL = "mongodb://root:rootpassword@mongo:27017/integr8scode?authSource=admin" - KUBERNETES_CONFIG_PATH = "/app/kubeconfig.yaml" KUBERNETES_CA_CERTIFICATE_PATH = "/app/certs/k8s-ca.pem" K8S_POD_CPU_LIMIT = "1000m" diff --git a/backend/secrets.example.toml b/backend/secrets.example.toml index 36b1823b..31498a42 100644 --- a/backend/secrets.example.toml +++ b/backend/secrets.example.toml @@ -13,5 +13,5 @@ # # Kubernetes: store as a Secret and mount at /app/secrets.toml -SECRET_KEY = "CHANGE_ME_min_32_chars_long_!!!!" -MONGODB_URL = "mongodb://user:password@mongo:27017/integr8scode?authSource=admin" +SECRET_KEY = "uS5xBF-OKXHV-1vqU4ASLwyPcKpSdUTLqGHPYs3y-Yc" +MONGODB_URL = "mongodb://root:rootpassword@mongo:27017/integr8scode?authSource=admin" diff --git a/docs/getting-started.md b/docs/getting-started.md index 16d1b804..0b9348b3 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,9 +11,12 @@ The full stack includes a Svelte frontend, FastAPI backend, MongoDB, Redis, Kafk ```bash git clone https://github.com/HardMax71/Integr8sCode.git cd Integr8sCode +cp backend/secrets.example.toml backend/secrets.toml ./deploy.sh dev ``` +The `secrets.toml` file holds credentials (`SECRET_KEY`, `MONGODB_URL`) and is gitignored. The example template ships with working development defaults, so copying it is all you need for local use. + Wait for the services to come up. You can watch progress with `docker compose logs -f` in another terminal. When you see the backend responding to health checks, you're ready. Verify everything is running: @@ -111,4 +114,4 @@ docker compose down -v - [Architecture Overview](architecture/overview.md) - How the pieces fit together - [Deployment](operations/deployment.md) - Production deployment with Helm - [API Reference](reference/api-reference.md) - Full endpoint documentation -- [Environment Variables](reference/environment-variables.md) - Configuration options +- [Configuration Reference](reference/configuration.md) - TOML configuration options diff --git a/docs/index.md b/docs/index.md index 3eda1c13..a1364f17 100644 --- a/docs/index.md +++ b/docs/index.md @@ -81,11 +81,11 @@ automatic retries via dead letter queue for failed processing. Policies, network isolation, and supply chain -- :material-variable: **[Environment Variables](reference/environment-variables.md)** +- :material-cog: **[Configuration Reference](reference/configuration.md)** --- - Complete configuration reference + TOML configuration and secrets - :material-monitor: **[Frontend](frontend/routing.md)** diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 1dc7492f..d5a19adf 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -161,7 +161,7 @@ The Helm chart organizes templates by function: |-----------------------------|-------------------------------------------| | `templates/rbac/` | ServiceAccount, Role, RoleBinding | | `templates/secrets/` | Kubeconfig and Kafka JAAS | -| `templates/configmaps/` | Environment variables | +| `templates/configmaps/` | TOML configuration and environment | | `templates/infrastructure/` | Zookeeper, Kafka, Schema Registry, Jaeger | | `templates/app/` | Backend and Frontend deployments | | `templates/workers/` | All seven worker deployments | @@ -214,8 +214,7 @@ The `values.yaml` file contains all configurable options. Key sections: --8<-- "helm/integr8scode/values.yaml:10:19" ``` -Environment variables shared across services live in the `env` section and get rendered into a ConfigMap. -Service-specific overrides go in their respective sections. For example, to increase backend replicas and memory: +Application settings are loaded from TOML files (`config.toml` + `secrets.toml`), not environment variables. The Helm chart mounts these as files into each pod. Service-specific overrides go in their respective sections. For example, to increase backend replicas and memory: ```yaml backend: diff --git a/docs/reference/environment-variables.md b/docs/reference/configuration.md similarity index 53% rename from docs/reference/environment-variables.md rename to docs/reference/configuration.md index 0062a1ed..fd553ff1 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/configuration.md @@ -1,36 +1,61 @@ -# Environment Variables +# Configuration Reference -All configuration is done through environment variables loaded from `.env` files. Copy `backend/.env` to get started and adjust as needed. +All backend configuration is loaded from TOML files — no environment variables, no `.env` files. The `Settings` class (`app/settings.py`) reads files in this order, each layer overriding the previous: -The code blocks below show the actual `backend/.env` development configuration. The legend tables show what the app uses if a variable is **not set at all** — these are the `settings.py` defaults, which may differ from the dev config. +1. **`config.toml`** — base settings, committed to git (no secrets) +2. **`secrets.toml`** — sensitive values (`SECRET_KEY`, `MONGODB_URL`), gitignored +3. **per-worker override** — optional TOML file for service-specific settings (e.g. `config.coordinator.toml`) -## Core +```python +# Default — reads config.toml + secrets.toml +Settings() + +# Tests — reads config.test.toml + secrets.toml +Settings(config_path="config.test.toml") + +# Worker — reads config.toml + secrets.toml + worker override +Settings(override_path="config.coordinator.toml") +``` + +## Secrets + +Credentials live in `secrets.toml`, which is gitignored. A committed template with development defaults is provided: ```bash ---8<-- "backend/.env:1:9" +cp backend/secrets.example.toml backend/secrets.toml +``` + +```toml +--8<-- "backend/secrets.example.toml" +``` + +For production, mount `secrets.toml` from a Kubernetes Secret at `/app/secrets.toml`. In CI, generate it from repository secrets (see the template comments for an example). + +## Core + +```toml +--8<-- "backend/config.toml:1:9" ``` ??? info "Legend" - | Variable | Description | If unset | - |----------|-------------|---------| + | Key | Description | Default | + |-----|-------------|---------| | `PROJECT_NAME` | Application name for logs and metadata | `integr8scode` | | `DATABASE_NAME` | MongoDB database name | `integr8scode_db` | - | `SECRET_KEY` | JWT signing key. **Required**, min 32 chars | — | + | `SECRET_KEY` | JWT signing key, min 32 chars. **Lives in `secrets.toml`** | — (required) | | `ALGORITHM` | JWT signing algorithm | `HS256` | | `ACCESS_TOKEN_EXPIRE_MINUTES` | Token lifetime in minutes | `1440` (24h) | - | `MONGO_ROOT_USER` | MongoDB admin username | `root` | - | `MONGO_ROOT_PASSWORD` | MongoDB admin password | `rootpassword` | - | `MONGODB_URL` | MongoDB connection string | `mongodb://mongo:27017/integr8scode` | + | `MONGODB_URL` | MongoDB connection string. **Lives in `secrets.toml`** | `mongodb://mongo:27017/integr8scode` | ## Kubernetes -```bash ---8<-- "backend/.env:10:17" +```toml +--8<-- "backend/config.toml:11:20" ``` ??? info "Legend" - | Variable | Description | If unset | - |----------|-------------|---------| + | Key | Description | Default | + |-----|-------------|---------| | `KUBERNETES_CONFIG_PATH` | Path to kubeconfig | `~/.kube/config` | | `KUBERNETES_CA_CERTIFICATE_PATH` | Custom CA cert for K8s API | — | | `K8S_POD_CPU_LIMIT` | CPU limit per pod | `1000m` | @@ -43,13 +68,13 @@ The code blocks below show the actual `backend/.env` development configuration. ## Kafka -```bash ---8<-- "backend/.env:19:31" +```toml +--8<-- "backend/config.toml:26:37" ``` ??? info "Legend" - | Variable | Description | If unset | - |----------|-------------|---------| + | Key | Description | Default | + |-----|-------------|---------| | `KAFKA_BOOTSTRAP_SERVERS` | Broker addresses (comma-separated) | `kafka:29092` | | `SCHEMA_REGISTRY_URL` | Confluent Schema Registry URL | `http://schema-registry:8081` | | `ENABLE_EVENT_STREAMING` | Enable Kafka events | `false` | @@ -64,41 +89,41 @@ The code blocks below show the actual `backend/.env` development configuration. ## SSE (Server-Sent Events) -```bash ---8<-- "backend/.env:32:34" +```toml +--8<-- "backend/config.toml:39:40" ``` ??? info "Legend" - | Variable | Description | If unset | - |----------|-------------|---------| + | Key | Description | Default | + |-----|-------------|---------| | `SSE_CONSUMER_POOL_SIZE` | Number of Kafka consumers for SSE streaming | `10` | | `SSE_HEARTBEAT_INTERVAL` | Keepalive interval in seconds | `30` | ## Tracing (OpenTelemetry) -```bash ---8<-- "backend/.env:39:45" +```toml +--8<-- "backend/config.toml:45:51" ``` ??? info "Legend" - | Variable | Description | If unset | - |----------|-------------|---------| + | Key | Description | Default | + |-----|-------------|---------| | `ENABLE_TRACING` | Enable distributed tracing | `true` | | `JAEGER_AGENT_HOST` | Jaeger agent hostname | `jaeger` | | `JAEGER_AGENT_PORT` | Jaeger agent UDP port | `6831` | | `TRACING_SERVICE_NAME` | Service name in traces | `integr8scode-backend` | | `TRACING_SERVICE_VERSION` | Version in trace metadata | `1.0.0` | - | `TRACING_SAMPLING_RATE` | Sample rate (0.0–1.0) | `0.1` | + | `TRACING_SAMPLING_RATE` | Sample rate (0.0-1.0) | `0.1` | ## Dead Letter Queue -```bash ---8<-- "backend/.env:47:53" +```toml +--8<-- "backend/config.toml:53:59" ``` ??? info "Legend" - | Variable | Description | If unset | - |----------|-------------|---------| + | Key | Description | Default | + |-----|-------------|---------| | `DLQ_RETRY_MAX_ATTEMPTS` | Retries before archiving | `5` | | `DLQ_RETRY_BASE_DELAY_SECONDS` | Base delay between retries | `60` | | `DLQ_RETRY_MAX_DELAY_SECONDS` | Max delay (backoff cap) | `3600` | @@ -108,13 +133,13 @@ The code blocks below show the actual `backend/.env` development configuration. ## Service & OTEL -```bash ---8<-- "backend/.env:55:66" +```toml +--8<-- "backend/config.toml:61:70" ``` ??? info "Legend" - | Variable | Description | If unset | - |----------|-------------|---------| + | Key | Description | Default | + |-----|-------------|---------| | `APP_URL` | Public URL for notifications | `https://integr8scode.cc` | | `SERVICE_NAME` | Service identifier | `integr8scode-backend` | | `SERVICE_VERSION` | Service version | `1.0.0` | @@ -125,13 +150,13 @@ The code blocks below show the actual `backend/.env` development configuration. ## Server -```bash ---8<-- "backend/.env:68:84" +```toml +--8<-- "backend/config.toml:72:80" ``` ??? info "Legend" - | Variable | Description | If unset | - |----------|-------------|---------| + | Key | Description | Default | + |-----|-------------|---------| | `WEB_CONCURRENCY` | Gunicorn worker processes | `4` | | `WEB_THREADS` | Threads per worker | `1` | | `WEB_TIMEOUT` | Request timeout in seconds | `60` | @@ -139,3 +164,31 @@ The code blocks below show the actual `backend/.env` development configuration. | `SERVER_HOST` | Bind address | `localhost` | | `BCRYPT_ROUNDS` | Password hashing rounds | `12` | | `REDIS_MAX_CONNECTIONS` | Redis connection pool size | `200` | + +## Worker overrides + +Each worker runs with a small override TOML that sets `TRACING_SERVICE_NAME` and `KAFKA_CONSUMER_GROUP_ID`. These are mounted alongside `config.toml` and `secrets.toml` in Docker Compose: + +| File | Service | +|------|---------| +| `config.coordinator.toml` | Execution coordinator | +| `config.k8s-worker.toml` | Kubernetes pod manager | +| `config.pod-monitor.toml` | Pod status watcher | +| `config.result-processor.toml` | Result processor | +| `config.saga-orchestrator.toml` | Saga orchestrator | +| `config.event-replay.toml` | Event replay | +| `config.dlq-processor.toml` | Dead letter queue processor | + +## Test configuration + +`config.test.toml` is a full config file tuned for fast test execution (lower bcrypt rounds, relaxed rate limits, shorter Kafka timeouts). Tests load it with: + +```python +Settings(config_path="config.test.toml") +``` + +Secrets are still loaded from `secrets.toml`. In CI, the workflow copies the example template: + +```bash +cp backend/secrets.example.toml backend/secrets.toml +``` diff --git a/mkdocs.yml b/mkdocs.yml index a1bf4c7d..b3fc0fae 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -128,7 +128,7 @@ nav: - Service Lifecycle: architecture/lifecycle.md - API Reference: reference/api-reference.md - - Environment Variables: reference/environment-variables.md + - Configuration Reference: reference/configuration.md - Components: - SSE: From b5526f1ae5d749b85fb2f88392e62c0a1abf5e91 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 30 Jan 2026 22:05:26 +0100 Subject: [PATCH 4/5] fixes --- .github/workflows/docs.yml | 3 +++ .github/workflows/stack-tests.yml | 3 +++ backend/secrets.example.toml | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7d40a544..bac34726 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,6 +42,9 @@ jobs: - name: Install backend dependencies run: cd backend && uv sync --frozen + - name: Set up config for OpenAPI generation + run: cp backend/secrets.example.toml backend/secrets.toml + - name: Generate OpenAPI spec run: ./deploy.sh openapi diff --git a/.github/workflows/stack-tests.yml b/.github/workflows/stack-tests.yml index b1ef9801..c2804f73 100644 --- a/.github/workflows/stack-tests.yml +++ b/.github/workflows/stack-tests.yml @@ -48,6 +48,9 @@ jobs: uv python install 3.12 uv sync --frozen + - name: Set up config + run: cp backend/secrets.example.toml backend/secrets.toml + - name: Run unit tests timeout-minutes: 5 run: | diff --git a/backend/secrets.example.toml b/backend/secrets.example.toml index 31498a42..0f7b41ef 100644 --- a/backend/secrets.example.toml +++ b/backend/secrets.example.toml @@ -13,5 +13,5 @@ # # Kubernetes: store as a Secret and mount at /app/secrets.toml -SECRET_KEY = "uS5xBF-OKXHV-1vqU4ASLwyPcKpSdUTLqGHPYs3y-Yc" +SECRET_KEY = "CHANGE_ME_min_32_chars_long_!!!!" MONGODB_URL = "mongodb://root:rootpassword@mongo:27017/integr8scode?authSource=admin" From bbf99d9d0659730da9dce18c2c352e9755aff978 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Fri, 30 Jan 2026 22:15:09 +0100 Subject: [PATCH 5/5] docker compose fix --- docker-compose.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 51af11d9..bbbb40a1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -378,6 +378,9 @@ services: environment: - KAFKA_BOOTSTRAP_SERVERS=kafka:29092 - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + volumes: + - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro command: ["python", "-m", "scripts.create_topics"] networks: - app-network @@ -397,9 +400,11 @@ services: mongo: condition: service_healthy environment: - - MONGODB_URL=mongodb://${MONGO_ROOT_USER:-root}:${MONGO_ROOT_PASSWORD:-rootpassword}@mongo:27017/integr8scode?authSource=admin - DEFAULT_USER_PASSWORD=${DEFAULT_USER_PASSWORD:-user123} - ADMIN_USER_PASSWORD=${ADMIN_USER_PASSWORD:-admin123} + volumes: + - ./backend/config.toml:/app/config.toml:ro + - ./backend/secrets.toml:/app/secrets.toml:ro command: ["python", "-m", "scripts.seed_users"] networks: - app-network