ci.yml

name: CI

on:
  push:
    branches: [main]
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # ---------------------------------------------------------------------------
  # Detect which paths changed so downstream jobs can be skipped when
  # irrelevant files are touched.  Always runs — no path filter here.
  # ---------------------------------------------------------------------------
  changes:
    name: Detect changed paths
    runs-on: ubuntu-22.04
    permissions:
      contents: read
      pull-requests: read
    outputs:
      python: ${{ steps.filter.outputs.python }}
      infra: ${{ steps.filter.outputs.infra }}
      proto: ${{ steps.filter.outputs.proto }}
      ts_sdk: ${{ steps.filter.outputs.ts_sdk }}
      admin_ui: ${{ steps.filter.outputs.admin_ui }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v4
        id: filter
        with:
          filters: |
            python:
              - 'packages/**'
              - 'apps/gateway/**'
              - 'tests/**'
              - 'scripts/**'
              - 'sdks/python/**'
              - 'pyproject.toml'
              - 'uv.lock'
              - 'proto/**'
              - 'buf.yaml'
              - 'buf.gen.yaml'
            infra:
              - 'infra/terraform/**'
              - 'infra/helm/**'
            proto:
              - 'proto/**'
              - 'buf.yaml'
              - 'buf.gen.yaml'
            ts_sdk:
              - 'sdks/typescript/**'
            admin_ui:
              - 'apps/admin-ui/**'

  # ---------------------------------------------------------------------------
  # Lint + type-check + unit tests (Ubuntu + macOS + Windows)
  # Cross-platform requirement: every step must pass on all three OSes.
  # ---------------------------------------------------------------------------
  lint-test:
    name: Lint & Test (${{ matrix.os }})
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-22.04, macos-14, windows-latest]

    steps:
      - uses: actions/checkout@v4

      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          version: "latest"
          enable-cache: true
          cache-dependency-glob: "uv.lock"

      - name: Set up Python 3.12
        run: uv python install 3.12

      - name: Install Python dependencies
        run: uv sync --all-packages

      - name: Ruff lint
        run: uv run ruff check .

      - name: Ruff format check
        run: uv run ruff format --check .

      - name: RAG001 — logging policy check
        run: uv run python scripts/check_logging.py

      - name: Mypy (strict)
        run: uv run mypy packages/ apps/gateway/

      - name: Pytest
        # The perf gate is excluded here — it runs as its own single-runner job
        # (perf-gate) where timing is stable; noisier macOS / Windows runners
        # would make a latency budget flaky.  The red-team gate is likewise its
        # own job (redteam-gate) so the security suite is a named exit gate.
        run: uv run pytest tests/ packages/ sdks/python/tests -x -q --tb=short -m "not perf and not redteam"

  # ---------------------------------------------------------------------------
  # Secrets scan — blocks if a real credential is planted in a PR.
  # Intentionally has no path filter: always run on every push/PR.
  # ---------------------------------------------------------------------------
  secrets-scan:
    name: Secrets scan (gitleaks)
    runs-on: ubuntu-22.04
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install gitleaks
        run: |
          curl -sSfL \
            https://github.com/gitleaks/gitleaks/releases/download/v8.21.2/gitleaks_8.21.2_linux_x64.tar.gz \
            | tar -xz gitleaks
          sudo mv gitleaks /usr/local/bin/gitleaks
      - name: Run gitleaks
        run: gitleaks detect --source . --redact --exit-code 1

  # ---------------------------------------------------------------------------
  # Schema-drift gate — regenerates dist/schemas/ and fails if anything in the
  # tracked schema artifacts is modified, staged, or untracked. Catches both
  # "forgot to regenerate" and "forgot to git add" mistakes.
  # ---------------------------------------------------------------------------
  schema-drift:
    name: Schema drift (dist/schemas/)
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
      - run: uv sync --all-packages
      - name: Regenerate JSON schemas
        env:
          PYTHONPATH: packages/core/src
        run: uv run python -m rag_core.gen_schemas dist/schemas/
      - name: Verify no drift
        run: uv run python scripts/check_schema_drift.py

  # ---------------------------------------------------------------------------
  # OpenAPI-drift gate (Step 3.7) — re-export dist/openapi.{json,yaml} from the
  # live gateway routes and fail if the committed spec is stale.  The spec is
  # the source of truth for the REST-side SDKs, so it must never lag the app.
  # ---------------------------------------------------------------------------
  openapi-drift:
    name: OpenAPI drift (dist/openapi.*)
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
      - run: uv sync --all-packages
      - name: Re-export OpenAPI spec
        run: uv run python scripts/export_openapi.py
      - name: Verify no drift
        run: uv run python scripts/check_openapi_drift.py

  # ---------------------------------------------------------------------------
  # TypeScript SDK (Step 3.7) — typecheck + unit tests for the hand-written
  # @agentcontextos/sdk client.  Runs only when sdks/typescript changes.
  # ---------------------------------------------------------------------------
  ts-sdk:
    name: TypeScript SDK (typecheck + test)
    needs: changes
    if: needs.changes.outputs.ts_sdk == 'true'
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: sdks/typescript
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v4
        with:
          version: 9
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: pnpm
          cache-dependency-path: pnpm-lock.yaml
      - run: pnpm install --frozen-lockfile
      - name: Typecheck
        run: pnpm run typecheck
      - name: Test
        run: pnpm test
      - name: Build
        run: pnpm run build

  # ---------------------------------------------------------------------------
  # Admin UI (Step 3.10) — typecheck + lint + unit tests + production build for
  # the Next.js operator console.  Runs only when apps/admin-ui changes.
  # ---------------------------------------------------------------------------
  admin-ui:
    name: Admin UI (typecheck + test + build)
    needs: changes
    if: needs.changes.outputs.admin_ui == 'true'
    runs-on: ubuntu-22.04
    defaults:
      run:
        working-directory: apps/admin-ui
    steps:
      - uses: actions/checkout@v4
      - uses: pnpm/action-setup@v4
        with:
          version: 9
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: pnpm
          cache-dependency-path: pnpm-lock.yaml
      - run: pnpm install --frozen-lockfile
      - name: Typecheck
        run: pnpm run typecheck
      - name: Lint
        run: pnpm run lint
      - name: Test
        run: pnpm test
      - name: Build
        run: pnpm run build

  # ---------------------------------------------------------------------------
  # Framework adapters (Step 3.8) — install the lighter framework extras and
  # run the agentcontextos.integrations adapter tests so the mapping logic is
  # exercised against the real framework base classes (not just skipped).  The
  # heavier extras (crewai, semantic-kernel) stay skip-if-absent here.
  # ---------------------------------------------------------------------------
  integrations:
    name: Framework adapters
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
      - run: uv sync --all-packages
      - name: Install framework extras
        run: uv pip install langchain-core llama-index-core haystack-ai dspy langgraph autogen-core
      - name: Run adapter tests
        run: uv run pytest sdks/python/tests/test_integrations.py -q

  # ---------------------------------------------------------------------------
  # Proto lint + drift gate (Step 3.2) — buf lint over proto/, then
  # regenerate Python stubs and fail if anything in apps/gateway/.../_grpc_gen/
  # differs from a fresh codegen run.  Drift gate runs on all three OSes to
  # catch grpcio-tools / mypy-protobuf platform-specific output differences
  # before they hit a developer.
  # ---------------------------------------------------------------------------
  proto-lint:
    name: Proto lint (buf)
    needs: changes
    if: needs.changes.outputs.proto == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: bufbuild/buf-setup-action@v1
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
      - run: buf lint

  proto-drift:
    name: Proto stub drift (${{ matrix.os }})
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-22.04, macos-14, windows-latest]
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
      - run: uv sync --all-packages
      - name: Regenerate gRPC stubs
        run: uv run python scripts/gen_proto_py.py
      - name: Verify no drift
        run: uv run python scripts/check_proto_drift.py

  # ---------------------------------------------------------------------------
  # Logging gates — schema + PII + event-registry (activated in Step 0.7b)
  # ---------------------------------------------------------------------------
  log-gates:
    name: Log schema + PII gates
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
      - run: uv sync --all-packages
      - run: uv run pytest tests/logs/ -v

  # ---------------------------------------------------------------------------
  # Gateway latency gate (Step 4.6) — the p99 ≤ 30 ms overhead budget.
  # Single runner (ubuntu) on purpose: a latency budget needs stable timing, so
  # it is excluded from the noisier cross-OS unit sweep and enforced here.
  # ---------------------------------------------------------------------------
  perf-gate:
    name: Gateway latency gate
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
      - run: uv sync --all-packages
      - name: Gateway p99 overhead gate
        run: uv run pytest tests/perf/ -v -m perf

  # ---------------------------------------------------------------------------
  # Red-team — adversarial security probe suites (Step 7.3).  Deterministic +
  # in-process, so a single runner; named separately so the security gate is a
  # first-class CI exit gate (prompt injection ≥95% block + no untrusted chunk
  # in a system-trust position; ACL bypass; PII egress; tenant escape).
  # ---------------------------------------------------------------------------
  redteam-gate:
    name: Red-team security gate
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
      - run: uv sync --all-packages
      - name: Adversarial probe suites
        run: uv run pytest tests/redteam/ -v -m redteam

  # ---------------------------------------------------------------------------
  # IaC validation — terraform validate + helm lint (no live cluster needed)
  # ---------------------------------------------------------------------------
  iac:
    name: IaC validate (terraform + helm)
    needs: changes
    if: needs.changes.outputs.infra == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v3
        with:
          terraform_version: "~1.9"

      - name: Setup Helm
        uses: azure/setup-helm@v4
        with:
          version: "latest"

      - name: Terraform validate — dev
        working-directory: infra/terraform/environments/dev
        run: |
          terraform init -backend=false
          terraform validate

      - name: Terraform validate — prod
        working-directory: infra/terraform/environments/prod
        run: |
          terraform init -backend=false
          terraform validate

      - name: Helm lint — rag-platform
        run: helm lint infra/helm/rag-platform/

      - name: Helm template — dry run
        run: helm template rag-platform infra/helm/rag-platform/ > /dev/null

  # ---------------------------------------------------------------------------
  # Dependency CVE audit
  # ---------------------------------------------------------------------------
  audit:
    name: pip-audit
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: "uv.lock"
      - run: uv sync --all-packages
      - run: uv run pip-audit

  # ---------------------------------------------------------------------------
  # Required status-check gate.
  # Always runs; fails only if a real job failed (skipped jobs are OK).
  # Configure "CI passed" as the required branch-protection check in GitHub.
  # ---------------------------------------------------------------------------
  ci-pass:
    name: CI passed
    needs: [lint-test, secrets-scan, schema-drift, log-gates, audit, iac, proto-lint, proto-drift, perf-gate, redteam-gate]
    if: always()
    runs-on: ubuntu-22.04
    steps:
      - name: Check job results
        run: |
          results='${{ toJSON(needs.*.result) }}'
          echo "Job results: $results"
          if echo "$results" | grep -q '"failure"'; then
            echo "One or more jobs failed."
            exit 1
          fi
          echo "All jobs passed or were skipped."