graph.yml

# Graph Database Deployment Configuration
# This file defines all graph database writer configurations for both environments
#
# ARCHITECTURE: LadybugDB-only graph tier system
# - All tiers use LadybugDB backend (embedded graph database)
# - Tiers differentiated by: instance isolation, subgraph support, resource allocation
# - Platform metadata stored in PostgreSQL, graphs contain only business data
# - Supports "MotherDuck for Graphs" pattern: DuckDB staging → LadybugDB materialization
#
# TIER PHILOSOPHY:
# - Standard: Dedicated m7g.large, single database + 3 subgraphs, cost-efficient entry
# - Large: Dedicated r7g.large, subgraphs for organization, team collaboration
# - XLarge: Dedicated r7g.xlarge, maximum subgraphs, advanced scale
# - Shared: Public data repositories (SEC) - separate infrastructure

production:
  writers:
    # =========================================================================
    # LADYBUGDB STANDARD TIER - Dedicated m7g.large
    # =========================================================================
    - name: ladybug-standard
      description: Dedicated m7g.large LadybugDB tier for cost-efficient entry with subgraph support
      backend: ladybug
      stack_suffix: LadybugStandard
      tier: ladybug-standard

      # Subgraph configuration
      max_subgraphs: 3 # Up to 3 subgraphs (1 parent + 3 children = 4 databases total)

      # Resource limits
      api_rate_multiplier: 1.0 # Base rate limits (1x)

      # Copy operation limits
      copy_operations:
        max_file_size_gb: 1.0 # Maximum file size per copy operation
        timeout_seconds: 900 # 15 minute timeout for dedicated instance
        concurrent_operations: 1 # One copy operation at a time
        max_files_per_operation: 100 # Maximum files per batch copy
        daily_copy_operations: 25 # Daily copy operation limit

      # Backup limits
      backup_limits:
        max_backup_size_gb: 10 # Maximum backup size for standard tier
        backup_retention_days: 7 # Backup retention period
        max_backups_per_day: 2 # Daily backup limit

      # Graph content limits
      graph_limits:
        instance_storage_limit_gb: 20 # Soft storage cap for entire instance (parent + subgraphs + staging)
        max_rows_per_copy: 2000000   # 2M rows per copy operation (hard limit, prevents OOM during materialization)
        max_single_table_rows: 5000000 # 5M rows per staging table
        chunk_size_rows: 500000      # 500K rows per materialization chunk (conservative for 2GB buffer pool)
        warn_at_percentage: 80       # Warn when approaching storage limit

      # Memory boost configuration
      memory:
        baseline_mb: 2048    # Default LadybugDB memory (m7g.large = 8GB)
        max_boost_mb: 6144   # Max boost during materialization
        auto_reset: true     # Reset to baseline after operation

      # Instance configuration
      instance:
        type: m7g.large # 8GB RAM, 2 vCPUs, ARM64 - dedicated per customer
        databases_per_instance: 1 # Dedicated: one customer per instance

        # Memory configuration
        max_memory_mb: 6144 # 6GB available (8GB - 2GB OS overhead)
        memory_per_db_mb: 2048 # 2GB per database

        # LadybugDB-specific performance settings
        chunk_size: 1000 # Batch operation chunk size
        query_timeout: 30 # Query timeout in seconds
        max_query_length: 10000 # Maximum query size in characters
        ingestion_batch_size: 1000 # Batch size for data ingestion
        connection_pool_size: 10 # Max concurrent connections per database

        # DuckDB staging configuration
        duckdb_memory_limit: "2GB" # Conservative for standard tier
        duckdb_max_threads: 2 # m7g.large = 2 vCPU

      # Deployment configuration
      deployment:
        # Standard tier is always deployed - no enable_var needed
        always_enabled: true # LadybugDB standard tier is the foundation - always available

    # =========================================================================
    # LADYBUGDB LARGE TIER - Dedicated r7g.large
    # =========================================================================
    - name: ladybug-large
      description: Dedicated r7g.large with subgraph support (10 subgraphs)
      backend: ladybug
      stack_suffix: LadybugLarge
      tier: ladybug-large

      # Subgraph configuration
      max_subgraphs: 10 # Support up to 10 subgraphs (1 parent + 10 children = 11 databases total)

      # Resource limits
      api_rate_multiplier: 1.5 # 1.5x rate limits (2x RAM, same CPU as standard)

      # Copy operation limits
      copy_operations:
        max_file_size_gb: 5.0 # Larger files for professional workloads
        timeout_seconds: 1800 # 30 minute timeout for large operations
        concurrent_operations: 3 # Support concurrent operations for team
        max_files_per_operation: 1000 # Higher batch limits
        daily_copy_operations: 50 # More operations for active development

      # Backup limits
      backup_limits:
        max_backup_size_gb: 50 # Larger backup capacity for professional workloads
        backup_retention_days: 30 # Extended retention for team collaboration
        max_backups_per_day: 10 # More frequent backups for active development

      # Graph content limits
      graph_limits:
        instance_storage_limit_gb: 50 # Soft storage cap for entire instance (parent + subgraphs + staging)
        max_rows_per_copy: 10000000  # 10M rows per copy operation (hard limit, prevents OOM during materialization)
        max_single_table_rows: 25000000 # 25M rows per staging table
        chunk_size_rows: 1000000     # 1M rows per materialization chunk (conservative for 1.3GB buffer pool)
        warn_at_percentage: 80       # Warn when approaching storage limit

      # Memory boost configuration
      memory:
        baseline_mb: 4096    # Default LadybugDB memory (r7g.large = 16GB)
        max_boost_mb: 12288  # Max boost during materialization
        auto_reset: true     # Reset to baseline after operation

      # Instance configuration
      instance:
        type: r7g.large # 16GB RAM, 2 vCPUs, ARM64 - dedicated per customer
        databases_per_instance: 1 # One parent database per instance (subgraphs on same instance)

        # Memory configuration
        max_memory_mb: 14336 # 14GB total available for parent + subgraphs
        memory_per_db_mb: 1300 # ~1.3GB average per database (14GB / 11 databases)
        # Note: Memory is dynamically shared among parent and subgraphs

        # LadybugDB-specific performance settings
        chunk_size: 1500 # Larger chunks for better performance
        query_timeout: 60 # 60 second timeout for complex queries
        max_query_length: 25000 # Support more complex queries
        ingestion_batch_size: 2500 # Faster ingestion with dedicated resources
        connection_pool_size: 25 # Higher concurrency for team usage

        # DuckDB staging configuration
        duckdb_memory_limit: "4GB" # More memory for dedicated instance staging
        duckdb_max_threads: 2 # Match vCPU count (r7g.large = 2 vCPU)

      # Deployment configuration
      deployment:
        enable_var: LBUG_LARGE_ENABLED_PROD
        enabled_default: false

    # =========================================================================
    # LADYBUGDB XLARGE TIER - Dedicated r7g.xlarge
    # =========================================================================
    - name: ladybug-xlarge
      description: Dedicated r7g.xlarge with maximum subgraph support (25 subgraphs)
      backend: ladybug
      stack_suffix: LadybugXlarge
      tier: ladybug-xlarge

      # Subgraph configuration
      max_subgraphs: 25 # Practical limit of 25 subgraphs (marketed as "unlimited")
      # Note: 1 parent + 25 subgraphs = 26 databases total on xlarge instance

      # Resource limits
      api_rate_multiplier: 2.5 # 2.5x rate limits (4x RAM, 2x CPU vs standard)

      # Copy operation limits
      copy_operations:
        max_file_size_gb: 10.0 # Maximum file size for enterprise workloads
        timeout_seconds: 3600 # 1 hour timeout for massive operations
        concurrent_operations: 5 # Maximum concurrency for enterprise
        max_files_per_operation: 10000 # Very high batch limits
        daily_copy_operations: -1 # Unlimited for enterprise (no daily limit)

      # Backup limits
      backup_limits:
        max_backup_size_gb: 100 # Maximum backup capacity for enterprise
        backup_retention_days: 90 # Extended retention for compliance
        max_backups_per_day: -1 # Unlimited backups for enterprise (no daily limit)

      # Graph content limits
      graph_limits:
        instance_storage_limit_gb: 100 # Soft storage cap for entire instance (parent + subgraphs + staging)
        max_rows_per_copy: 50000000  # 50M rows per copy operation (hard limit, prevents OOM during materialization)
        max_single_table_rows: 100000000 # 100M rows per staging table
        chunk_size_rows: 5000000     # 5M rows per materialization chunk
        warn_at_percentage: 80       # Warn when approaching storage limit

      # Memory boost configuration
      memory:
        baseline_mb: 8192    # Default LadybugDB memory (r7g.xlarge = 32GB)
        max_boost_mb: 28672  # Max boost during materialization
        auto_reset: true     # Reset to baseline after operation

      # Instance configuration
      instance:
        type: r7g.xlarge # 32GB RAM, 4 vCPUs, ARM64 - large dedicated per customer
        databases_per_instance: 1 # One parent database per instance (many subgraphs on same instance)

        # Memory configuration
        max_memory_mb: 30720 # 30GB available (32GB - 2GB OS overhead)
        memory_per_db_mb: 1180 # ~1.2GB average per database (30GB / 26 databases)
        # Note: Memory is dynamically shared, some databases can use more when needed

        # LadybugDB-specific performance settings
        chunk_size: 2000 # Maximum chunk size for best performance
        query_timeout: 120 # 2 minute timeout for complex analytical queries
        max_query_length: 50000 # Support very complex queries
        ingestion_batch_size: 5000 # Fast bulk ingestion
        connection_pool_size: 50 # High concurrency for enterprise applications

        # DuckDB staging configuration
        duckdb_memory_limit: "8GB" # Large memory for enterprise staging workloads
        duckdb_max_threads: 4 # Match vCPU count (r7g.xlarge = 4 vCPU)

      # Deployment configuration
      deployment:
        enable_var: LBUG_XLARGE_ENABLED_PROD
        enabled_default: false

    # =========================================================================
    # LADYBUGDB SHARED TIER - Public Data Repositories
    # =========================================================================
    - name: ladybug-shared
      description: Shared public data repositories (SEC) - LadybugDB backend
      backend: ladybug
      stack_suffix: LadybugShared
      tier: ladybug-shared

      # Repository configuration
      max_subgraphs: 10 # Platform-managed subgraphs only (e.g., sec_historical)

      # Resource limits
      api_rate_multiplier: null # Separate rate limiting for public access

      # Copy operation limits (for platform data loading)
      copy_operations:
        max_file_size_gb: 10.0 # Large files for bulk data loading
        timeout_seconds: 3600 # 1 hour for massive SEC filings
        concurrent_operations: 3 # Moderate concurrency for platform operations
        max_files_per_operation: 10000 # High batch limits for quarterly filings
        daily_copy_operations: -1 # Unlimited for platform operations

      # Backup limits
      backup_limits:
        max_backup_size_gb: 100 # Large capacity for public data repositories
        backup_retention_days: 90 # Extended retention for public data
        max_backups_per_day: 5 # Moderate backup frequency

      # Instance configuration
      instance:
        type: r7g.2xlarge # 64GB RAM - required for large Fact table staging without dedup
        databases_per_instance: 1 # One repository per instance (SEC gets dedicated instance)
        # Note: Major repositories like SEC get dedicated instances for performance

        # Memory configuration - CONSERVATIVE DEFAULTS
        # During normal operation, both DuckDB and LadybugDB use low memory.
        # During staging/materialization, memory is temporarily boosted via overrides.
        # This prevents OOM when both systems are active (staging reads + materialization writes).
        #
        # Container has ~59GB limit (64GB instance - 2GB OS - kernel overhead)
        # Default: 10GB each leaves 39GB headroom for OS buffers, temp files, etc.
        # Boost: Staging gets 55GB DuckDB, Materialization gets 50GB LadybugDB
        max_memory_mb: 10240 # 10GB default for LadybugDB (boosted to 50GB during materialization)
        memory_per_db_mb: 10240 # 10GB per database default

        # Memory boost settings (used by staging/materialization code)
        duckdb_memory_boost: "55GB" # Applied during DuckDB staging operations
        ladybug_memory_boost_mb: 51200 # 50GB - Applied during LadybugDB materialization

        # LadybugDB-specific performance settings (optimized for read-heavy analytical queries)
        chunk_size: 2500 # Large chunks for bulk operations
        query_timeout: 300 # 5 minute timeout for complex analytical queries on public data
        max_query_length: 100000 # Very large queries supported for analytics
        ingestion_batch_size: 10000 # Fast bulk loading for public data updates
        connection_pool_size: 100 # High concurrency - many users accessing public data

        # DuckDB staging configuration - CONSERVATIVE DEFAULT
        # Default is low; boosted to duckdb_memory_boost during staging operations
        duckdb_memory_limit: "10GB"
        duckdb_max_threads: 4 # r7g.2xlarge = 8 vCPU, use 4 for DuckDB

      # Deployment configuration
      deployment:
        enable_var: LBUG_SHARED_ENABLED_PROD
        enabled_default: false # Opt-in: shared repository cluster

  # ===========================================================================
  # SHARED REPLICAS - Read-Only Fleet
  # ===========================================================================
  replicas:
    - name: shared-replicas
      description: Read replicas for shared repositories (SEC)
      template: graph-ladybug-replicas.yaml
      depends_on: ladybug-shared # Only deploy after shared master exists

      # Instance type (scaling/enabled via GHA variables: SHARED_REPLICAS_*_PROD)
      # Memory overrides for replica hardware (m7g.large = 8GB RAM)
      # These override the writer's 10GB settings which exceed container capacity
      instance:
        type: m7g.large # General purpose ARM64 - cost-effective for read-only (8GB RAM)
        max_memory_mb: 5120 # 5GB available (8GB - 2GB OS - 1GB app overhead)
        memory_per_db_mb: 3584 # 3.5GB for parent databases (sec)
        memory_per_subgraph_mb: 2048 # 2GB for subgraphs (sec_historical) - oversubscribed
        duckdb_memory_limit: "0" # No DuckDB queries on replicas
        duckdb_max_threads: 0 # No DuckDB queries on replicas

# =============================================================================
# STAGING ENVIRONMENT
# =============================================================================
staging:
  writers:
    # =========================================================================
    # LADYBUGDB STANDARD TIER - Staging
    # =========================================================================
    - name: ladybug-standard
      description: Dedicated m7g.large LadybugDB tier with subgraph support
      backend: ladybug
      stack_suffix: LadybugStandard
      tier: ladybug-standard

      max_subgraphs: 3
      api_rate_multiplier: 1.0

      # Copy operation limits
      copy_operations:
        max_file_size_gb: 1.0
        timeout_seconds: 900
        concurrent_operations: 1
        max_files_per_operation: 100
        daily_copy_operations: 25

      # Backup limits
      backup_limits:
        max_backup_size_gb: 10
        backup_retention_days: 7
        max_backups_per_day: 2

      # Graph content limits (same as production)
      graph_limits:
        max_nodes: 5000000
        max_relationships: 10000000
        max_rows_per_copy: 2000000
        max_single_table_rows: 5000000
        chunk_size_rows: 500000
        warn_at_percentage: 80

      # Memory boost configuration
      memory:
        baseline_mb: 2048
        max_boost_mb: 6144
        auto_reset: true

      instance:
        type: m7g.large # 8GB RAM, 2 vCPUs, ARM64 - aligned with production
        databases_per_instance: 1 # Dedicated
        max_memory_mb: 6144 # 6GB available (8GB - 2GB OS overhead)
        memory_per_db_mb: 2048 # 2GB per database
        chunk_size: 1000
        query_timeout: 30
        max_query_length: 10000
        ingestion_batch_size: 1000
        connection_pool_size: 10
        duckdb_memory_limit: "2GB" # Aligned with production
        duckdb_max_threads: 2

      deployment:
        # Standard tier is always deployed - no enable_var needed
        always_enabled: true

    # =========================================================================
    # LADYBUGDB LARGE TIER - Staging
    # =========================================================================
    - name: ladybug-large
      description: Dedicated r7g.large with subgraph support (10 subgraphs)
      backend: ladybug
      stack_suffix: LadybugLarge
      tier: ladybug-large

      max_subgraphs: 10
      api_rate_multiplier: 2.5

      copy_operations:
        max_file_size_gb: 5.0
        timeout_seconds: 1800
        concurrent_operations: 3
        max_files_per_operation: 1000
        daily_copy_operations: 50

      backup_limits:
        max_backup_size_gb: 50
        backup_retention_days: 30
        max_backups_per_day: 10

      # Graph content limits (same as production)
      graph_limits:
        max_nodes: 50000000
        max_relationships: 100000000
        max_rows_per_copy: 10000000
        max_single_table_rows: 25000000
        chunk_size_rows: 1000000
        warn_at_percentage: 80

      # Memory boost configuration
      memory:
        baseline_mb: 4096
        max_boost_mb: 12288
        auto_reset: true

      instance:
        type: r7g.large # 16GB RAM, 2 vCPUs, ARM64 - aligned with production
        databases_per_instance: 1
        max_memory_mb: 14336 # 14GB total available for parent + subgraphs
        memory_per_db_mb: 1300 # ~1.3GB average per database (14GB / 11 databases)
        chunk_size: 1500
        query_timeout: 60
        max_query_length: 25000
        ingestion_batch_size: 2500
        connection_pool_size: 25
        duckdb_memory_limit: "4GB" # Aligned with production
        duckdb_max_threads: 2 # r7g.large = 2 vCPU

      deployment:
        enable_var: LBUG_LARGE_ENABLED_STAGING
        enabled_default: false

    # =========================================================================
    # LADYBUGDB XLARGE TIER - Staging
    # =========================================================================
    - name: ladybug-xlarge
      description: Dedicated r7g.xlarge with maximum subgraph support (25 subgraphs)
      backend: ladybug
      stack_suffix: LadybugXlarge
      tier: ladybug-xlarge

      max_subgraphs: 25
      api_rate_multiplier: 5.0

      copy_operations:
        max_file_size_gb: 10.0
        timeout_seconds: 3600
        concurrent_operations: 5
        max_files_per_operation: 10000
        daily_copy_operations: -1

      backup_limits:
        max_backup_size_gb: 100
        backup_retention_days: 90
        max_backups_per_day: -1

      # Graph content limits (same as production)
      graph_limits:
        max_nodes: 200000000
        max_relationships: 500000000
        max_rows_per_copy: 50000000
        max_single_table_rows: 100000000
        chunk_size_rows: 5000000
        warn_at_percentage: 80

      # Memory boost configuration
      memory:
        baseline_mb: 8192
        max_boost_mb: 28672
        auto_reset: true

      instance:
        type: r7g.xlarge # 32GB RAM, 4 vCPUs, ARM64 - aligned with production
        databases_per_instance: 1
        max_memory_mb: 30720 # 30GB available (32GB - 2GB OS overhead)
        memory_per_db_mb: 1180 # ~1.2GB average per database (30GB / 26 databases)
        chunk_size: 2000
        query_timeout: 120
        max_query_length: 50000
        ingestion_batch_size: 5000
        connection_pool_size: 50
        duckdb_memory_limit: "8GB" # Aligned with production
        duckdb_max_threads: 4 # r7g.xlarge = 4 vCPU

      deployment:
        enable_var: LBUG_XLARGE_ENABLED_STAGING
        enabled_default: false

    # =========================================================================
    # LADYBUGDB SHARED TIER - Staging
    # =========================================================================
    - name: ladybug-shared
      description: Shared public data repositories (SEC) - LadybugDB backend
      backend: ladybug
      stack_suffix: LadybugShared
      tier: ladybug-shared

      max_subgraphs: 10 # Platform-managed subgraphs only
      api_rate_multiplier: null

      copy_operations:
        max_file_size_gb: 10.0
        timeout_seconds: 3600
        concurrent_operations: 3
        max_files_per_operation: 10000
        daily_copy_operations: -1

      backup_limits:
        max_backup_size_gb: 100
        backup_retention_days: 90
        max_backups_per_day: 5

      instance:
        type: r7g.large # 16GB RAM, 2 vCPU - scaled down from production r7g.2xlarge
        databases_per_instance: 1
        max_memory_mb: 5120 # 5GB default for LadybugDB
        memory_per_db_mb: 5120 # 5GB per database default
        duckdb_memory_boost: "12GB" # Applied during DuckDB staging operations (spill-to-disk for overflow)
        ladybug_memory_boost_mb: 12288 # 12GB - Applied during LadybugDB materialization
        chunk_size: 2000
        query_timeout: 300
        max_query_length: 100000
        ingestion_batch_size: 5000
        connection_pool_size: 50
        duckdb_memory_limit: "5GB"
        duckdb_max_threads: 2 # r7g.large = 2 vCPU

      deployment:
        enable_var: LBUG_SHARED_ENABLED_STAGING
        enabled_default: false # Opt-in: shared repository cluster

  # ===========================================================================
  # SHARED REPLICAS - Read-Only Fleet (Staging)
  # ===========================================================================
  replicas:
    - name: shared-replicas
      description: Read replicas for staging shared repositories
      template: graph-ladybug-replicas.yaml
      depends_on: ladybug-shared

      # Instance type (scaling/enabled via GHA variables: SHARED_REPLICAS_*_STAGING)
      # Memory overrides for replica hardware (m7g.large = 8GB RAM)
      instance:
        type: m7g.large # General purpose ARM64 - cost-effective for read-only (8GB RAM)
        max_memory_mb: 5120 # 5GB available (8GB - 2GB OS - 1GB app overhead)
        memory_per_db_mb: 3584 # 3.5GB for parent databases (sec)
        memory_per_subgraph_mb: 2048 # 2GB for subgraphs (sec_historical) - oversubscribed
        duckdb_memory_limit: "0" # No DuckDB queries on replicas
        duckdb_max_threads: 0 # No DuckDB queries on replicas

# =============================================================================
# DEVELOPMENT ENVIRONMENT
# =============================================================================
development:
  writers:
    # =========================================================================
    # LADYBUGDB STANDARD TIER - Development (subgraphs enabled for testing)
    # =========================================================================
    - name: ladybug-standard
      description: LadybugDB standard tier for local development with subgraph support
      backend: ladybug
      stack_suffix: LadybugDev
      tier: ladybug-standard

      max_subgraphs: 3
      api_rate_multiplier: 1.0

      copy_operations:
        max_file_size_gb: 1.0
        timeout_seconds: 900
        concurrent_operations: 1
        max_files_per_operation: 100
        daily_copy_operations: 25

      backup_limits:
        max_backup_size_gb: 10
        backup_retention_days: 7
        max_backups_per_day: 2

      # Graph content limits (same as production)
      graph_limits:
        max_nodes: 5000000
        max_relationships: 10000000
        max_rows_per_copy: 2000000
        max_single_table_rows: 5000000
        chunk_size_rows: 500000
        warn_at_percentage: 80

      # Memory boost configuration
      memory:
        baseline_mb: 2048
        max_boost_mb: 6144
        auto_reset: true

      instance:
        type: m7g.large
        databases_per_instance: 1 # Dedicated
        max_memory_mb: 6144
        memory_per_db_mb: 2048
        chunk_size: 750
        query_timeout: 30
        max_query_length: 10000
        ingestion_batch_size: 750
        connection_pool_size: 10
        duckdb_memory_limit: "2GB" # Default for local development
        duckdb_max_threads: 4 # Local dev can use more threads

      deployment:
        always_enabled: true
        enabled_default: true