docker-compose.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: texera-single-node
services:
  # Part1: Specification of the storage services used by Texera
  # MinIO is an S3-compatible object storage used to store datasets and files.
  minio:
    image: minio/minio:RELEASE.2025-02-28T09-55-16Z
    container_name: texera-minio
    ports:
      - "${MINIO_PORT:-9000}:9000"
    env_file:
      - .env
    volumes:
      - minio_data:/data
    command: server --console-address ":9001" /data

  # PostgreSQL with PGroonga extension for full-text search.
  # Used by lakeFS and Texera's metadata storage.
  postgres:
    image: groonga/pgroonga:4.0.1-debian-15
    container_name: texera-postgres
    restart: always
    env_file:
      - .env
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "texera", "-d", "texera_db"]
      interval: 10s
      retries: 5
      start_period: 5s
    volumes:
      - postgres_data:/var/lib/postgresql/data
      # mount the sql files for initializing the postgres
      - ./sql:/docker-entrypoint-initdb.d

  # lakeFS is the underlying storage of Texera's dataset service
  lakefs:
    image: treeverse/lakefs:1.51
    container_name: texera-lakefs
    restart: always
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_started
    env_file:
      - .env
    environment:
      # This port also need to be changed if the port of MinIO service is changed
      - LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT=${TEXERA_HOST}:${MINIO_PORT:-9000}
    entrypoint: ["/bin/sh", "-c"]
    command:
      - |
        lakefs setup --user-name "$LAKEFS_INSTALLATION_USER_NAME" --access-key-id "$LAKEFS_INSTALLATION_ACCESS_KEY_ID" --secret-access-key "$LAKEFS_INSTALLATION_SECRET_ACCESS_KEY" || true
        lakefs run &
        wait
    healthcheck:
      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8000/api/v1/healthcheck"]
      interval: 10s
      timeout: 5s
      retries: 10

  # Lakekeeper migration init container
  # This runs once to migrate the database before the lakekeeper server starts
  lakekeeper-migrate:
    image: vakamo/lakekeeper:v0.11.0
    container_name: texera-lakekeeper-migrate
    depends_on:
      postgres:
        condition: service_healthy
    env_file:
      - .env
    restart: "no"
    entrypoint: ["/home/nonroot/lakekeeper"]
    command: ["migrate"]

  # Lakekeeper is the Iceberg REST catalog service
  lakekeeper:
    image: vakamo/lakekeeper:v0.11.0
    container_name: texera-lakekeeper
    restart: always
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_started
      lakekeeper-migrate:
        condition: service_completed_successfully
    env_file:
      - .env
    entrypoint: ["/home/nonroot/lakekeeper"]
    command: ["serve"]
    ports:
      - "8181:8181"
    healthcheck:
      test: ["CMD", "/home/nonroot/lakekeeper", "healthcheck"]
      interval: 10s
      timeout: 5s
      retries: 10
      start_period: 10s

  lakekeeper-init:
    image: alpine:3.19
    container_name: texera-lakekeeper-init
    depends_on:
      lakekeeper:
        condition: service_healthy
      minio:
        condition: service_started
    env_file:
      - .env
    restart: "no"
    entrypoint: [ "/bin/sh", "-c" ]
    command:
      - |
        set -e
        
        echo "Installing dependencies..."
        apk add --no-cache curl ca-certificates
        
        echo "Installing MinIO Client..."
        wget -q https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc
        chmod +x /usr/local/bin/mc
        
        check_status() {
          if [ "$$1" -ge 200 ] && [ "$$1" -lt 300 ]; then
            echo "Created $$2 successfully (HTTP $$1)."
          elif [ "$$1" -eq 409 ]; then
            echo "$$2 already exists (HTTP 409). Treating as success."
          else
            echo "Failed to create $$2. HTTP Code: $$1"
            echo "ERROR RESPONSE:"
            if [ -f /tmp/response.txt ]; then cat /tmp/response.txt; fi
            echo ""
            exit 1
          fi
        }
        
        echo "Step 1: Initializing MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET'..."
        mc alias set minio "$$STORAGE_S3_ENDPOINT" "$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD" || true
        if mc ls minio/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET > /dev/null 2>&1; then
          echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' already exists."
        else
          mc mb minio/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET || {
            echo "Failed to create MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET'"
            exit 1
          }
          echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' created successfully."
        fi
        
        
        echo "Step 2: Initializing Default Project..."
        PROJECT_PAYLOAD='{"project-id": "00000000-0000-0000-0000-000000000000", "project-name": "default"}'
        
        PROJECT_CODE=$$(curl -s -o /tmp/response.txt -w "%{http_code}" \
            -X POST \
            -H "Content-Type: application/json" \
            -d "$$PROJECT_PAYLOAD" \
            "$$LAKEKEEPER_BASE_URI/management/v1/project" || echo "000")
        
        check_status "$$PROJECT_CODE" "Default Project"
        
        
        echo "Step 3: Initializing Warehouse '$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME'..."
        CREATE_PAYLOAD=$$(cat <<EOF
        {
          "warehouse-name": "$$STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME",
          "project-id": "00000000-0000-0000-0000-000000000000", 
          "storage-profile": {
            "type": "s3",
            "bucket": "$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET",
            "region": "$$STORAGE_ICEBERG_CATALOG_REST_REGION",
            "endpoint": "$$STORAGE_S3_ENDPOINT",
            "flavor": "s3-compat",
            "path-style-access": true,
            "sts-enabled": false
          },
          "storage-credential": {
            "type": "s3",
            "credential-type": "access-key",
            "aws-access-key-id": "$$STORAGE_S3_AUTH_USERNAME",
            "aws-secret-access-key": "$$STORAGE_S3_AUTH_PASSWORD"
          }
        }
        EOF
        )
        
        WAREHOUSE_CODE=$$(curl -s -o /tmp/response.txt -w "%{http_code}" \
            -X POST \
            -H "Content-Type: application/json" \
            -d "$$CREATE_PAYLOAD" \
            "$$LAKEKEEPER_BASE_URI/management/v1/warehouse" || echo "000")
        
        check_status "$$WAREHOUSE_CODE" "Lakekeeper Warehouse"
        
        echo "Initialization sequence completed successfully!"


  # Part2: Specification of Texera's micro-services
  # FileService provides endpoints for Texera's dataset management
  file-service:
    image: ${IMAGE_REGISTRY:-ghcr.io/apache}/texera-file-service:${IMAGE_TAG:-latest}
    container_name: file-service
    restart: always
    depends_on:
      minio:
        condition: service_started
      lakefs:
        condition: service_healthy
    env_file:
      - .env
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:9092/api/healthcheck"]
      interval: 5s
      timeout: 3s
      retries: 10

  # ConfigService provides endpoints for configuration management
  config-service:
    image: ${IMAGE_REGISTRY:-ghcr.io/apache}/texera-config-service:${IMAGE_TAG:-latest}
    container_name: config-service
    restart: always
    depends_on:
      postgres:
        condition: service_healthy
    env_file:
      - .env
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:9094/api/healthcheck"]
      interval: 5s
      timeout: 3s
      retries: 10

  # AccessControlService handles user permissions and access control
  access-control-service:
    image: ${IMAGE_REGISTRY:-ghcr.io/apache}/texera-access-control-service:${IMAGE_TAG:-latest}
    container_name: access-control-service
    restart: always
    depends_on:
      postgres:
        condition: service_healthy
    env_file:
      - .env
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:9096/api/healthcheck"]
      interval: 5s
      timeout: 3s
      retries: 10

  # WorkflowComputingUnitManagingService provides endpoints for managing computing units
  workflow-computing-unit-managing-service:
    image: ${IMAGE_REGISTRY:-ghcr.io/apache}/texera-workflow-computing-unit-managing-service:${IMAGE_TAG:-latest}
    container_name: workflow-computing-unit-managing-service
    restart: always
    depends_on:
      postgres:
        condition: service_healthy
    env_file:
      - .env
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:8888/api/healthcheck"]
      interval: 5s
      timeout: 3s
      retries: 10

  # WorkflowCompilingService provides endpoints for sanity check and schema propagation while workflows are being edited
  workflow-compiling-service:
    image: ${IMAGE_REGISTRY:-ghcr.io/apache}/texera-workflow-compiling-service:${IMAGE_TAG:-latest}
    container_name: workflow-compiling-service
    restart: always
    depends_on:
      file-service:
        condition: service_started
    env_file:
      - .env
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:9090/api/healthcheck"]
      interval: 5s
      timeout: 3s
      retries: 10

  # WorkflowRuntimeCoordinatorService provides endpoints for executing workflows and interactions during executions.
  workflow-runtime-coordinator-service:
    image: ${IMAGE_REGISTRY:-ghcr.io/apache}/texera-workflow-execution-coordinator:${IMAGE_TAG:-latest}
    container_name: workflow-runtime-coordinator-service
    restart: always
    depends_on:
      workflow-compiling-service:
        condition: service_started
      lakekeeper:
        condition: service_healthy
      lakekeeper-init:
        condition: service_completed_successfully
    env_file:
      - .env
    volumes:
      - workflow_result_data:/amber/user-resources

  # DashboardService provides endpoints for hub resource management.
  dashboard-service:
    image: ${IMAGE_REGISTRY:-ghcr.io/apache}/texera-dashboard-service:${IMAGE_TAG:-latest}
    container_name: dashboard-service
    restart: always
    depends_on:
      workflow-runtime-coordinator-service:
        condition: service_started
      workflow-compiling-service:
        condition: service_healthy
      file-service:
        condition: service_healthy
    env_file:
      - .env
    volumes:
      - workflow_result_data:/amber/user-resources
    healthcheck:
      test: ["CMD", "curl", "-sf", "http://localhost:8080/api/healthcheck"]
      interval: 5s
      timeout: 3s
      retries: 10

  # Part 3: reverse proxy service for Texera's micro services
  nginx:
    image: nginx:alpine
    container_name: texera-nginx
    depends_on:
      - workflow-compiling-service
      - file-service
      - dashboard-service
      - workflow-runtime-coordinator-service
      - config-service
      - access-control-service
      - workflow-computing-unit-managing-service
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "${TEXERA_PORT:-8080}:8080"

  startup-message:
    image: alpine:latest
    depends_on:
      nginx:
        condition: service_started
    environment:
      - TEXERA_PORT=${TEXERA_PORT:-8080}
    restart: "no"
    command: >
      sh -c '
      echo "";
      echo "=========================================";
      echo "  Texera has started successfully!";
      echo "  Access at: http://localhost:$$TEXERA_PORT";
      echo "=========================================";
      echo "";
      '

  # Part 4: Optional one-shot jobs
  # Loads example datasets and workflows into Texera on first startup.
  # Only runs when the "examples" profile is activated:
  #   docker compose --profile examples up
  example-data-loader:
    image: alpine:latest
    depends_on:
      dashboard-service:
        condition: service_healthy
      file-service:
        condition: service_healthy
    volumes:
      - ./examples:/examples:ro
    environment:
      - TEXERA_DASHBOARD_SERVICE_URL=http://dashboard-service:8080/api
      - TEXERA_FILE_SERVICE_URL=http://file-service:9092/api
      - TEXERA_EXAMPLE_USERNAME=${USER_SYS_ADMIN_USERNAME}
      - TEXERA_EXAMPLE_PASSWORD=${USER_SYS_ADMIN_PASSWORD}
    restart: "no"
    profiles:
      - examples
    command: >
      sh -c 'apk add --no-cache curl jq bash > /dev/null 2>&1 && bash /examples/load-examples.sh'

networks:
  default:
    name: texera-single-node

# persistent volumes
volumes:
  minio_data:
  postgres_data:
  workflow_result_data: