functional.yml

---
name: Functional and interoperability testing

on:
  workflow_dispatch:
  pull_request:
    paths:
      - 'Cargo.toml'
      - 'Cargo.lock'
      - '.github/workflows/functional.yml'
      - 'tests/**'
      - 'loadtest/**'
      - 'crates/**'
      - 'policy/**'
env:
  DATABASE_URL: postgresql://keystone:1234@127.0.0.1:5432/keystone
  KEYSTONE_URL: http://localhost:8080
  OPA_URL: http://localhost:8181
  OS_KEYSTONE_CONFIG_DIR: ${{ github.workspace }}/etc
  REGISTRY: ghcr.io


defaults:
  run:
    shell: bash

jobs:
  build:
    runs-on: ubuntu-latest
    steps:

      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - name: Enable cache
        uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
        with:
          path: |
            ~/.cache/pip
            ~/.cargo
          key: ${{ runner.os }}-integration

      - name: Rust Cache
        uses: swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0

      - name: Install Rust
        uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # stable
        with:
          toolchain: stable

      - name: Install protobuf-compiler
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler

      - name: Build Keystone
        run: cargo build --release

      - name: Move artifacts to the root
        run: mv target/release/keystone* ./

      - name: Setup OPA
        uses: open-policy-agent/setup-opa@v2
        with:
          version: latest

      - name: Upload built binaries
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: keystone
          path: |
            keystone
            keystone-db

  interop:
    runs-on: ubuntu-latest
    needs:
      - build
    env:
      OS_AUTH_URL: http://localhost:8080
      OS_USERNAME: admin
      OS_PASSWORD: password
      OS_USER_DOMAIN_ID: default
      OS_PROJECT_NAME: admin
      OS_PROJECT_DOMAIN_ID: default
    services:
      postgres:
        image: postgres:17
        env:
          POSTGRES_USER: keystone
          POSTGRES_PASSWORD: '1234'
        ports:
          - 5432:5432
        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - uses: ./.github/actions/deploy_keystone

      - name: Wait for py-keystone to bootstrap
        run: sleep 5

      - name: Test python keystone availability
        run: curl http://localhost:5001/v3

      - name: Test rust keystone availability
        run: curl http://localhost:8080/v3

      - name: Basic test using token validation
        run: |
          TOKEN1=$(osc --os-cloud admin auth login)
          TOKEN2=$(osc --os-cloud admin-rust auth login)

          echo "Use rust Keystone token against both Keystones"
          curl http://localhost:8080/v3/auth/tokens -H "X-Auth-Token: ${TOKEN1}" -H "X-Subject-Token: ${TOKEN1}" | jq
          curl http://localhost:5001/v3/auth/tokens -H "X-Auth-Token: ${TOKEN1}" -H "X-Subject-Token: ${TOKEN1}" | jq

          echo "Use python Keystone token against both Keystones"
          curl http://localhost:8080/v3/auth/tokens -H "X-Auth-Token: ${TOKEN2}" -H "X-Subject-Token: ${TOKEN2}" | jq
          curl http://localhost:5001/v3/auth/tokens -H "X-Auth-Token: ${TOKEN2}" -H "X-Subject-Token: ${TOKEN2}" | jq

      - name: Run api tests
        run: cargo test --test api

      # - name: Run interop tests
      #   run: cargo test --test interop

      - name: Dump py-keystone logs
        if: failure()
        run: docker logs keystone

      - name: Dump rust keystone log
        if: failure()
        run: cat rust.log

      - name: Dump OPA log
        if: failure()
        run: docker logs opa

  federation:
    runs-on: ubuntu-latest
    if: "github.actor != 'openstack-experimental-release-plz'"
    needs:
      - build
    env:
      KEYCLOAK_URL: http://localhost:8082
      BROWSERDRIVER_PORT: 4444
    services:
      postgres:
        image: postgres:17
        env:
          POSTGRES_USER: keystone
          POSTGRES_PASSWORD: '1234'
        ports:
          - 5432:5432
        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
      keycloak:
        image: ghcr.io/openstack-experimental/keystone/keycloak-ci-service:26.2
        env:
          KC_BOOTSTRAP_ADMIN_USERNAME: admin
          KC_BOOTSTRAP_ADMIN_PASSWORD: password
        ports:
          - 8082:8080
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - name: Install browser
        # This is crucial for Selenium to interact with a browser
        run: |
          sudo apt-get update
          sudo apt-get install -y firefox

      - uses: ./.github/actions/deploy_keystone

      - name: Start geckodriver for selenium
        run: /snap/bin/geckodriver --port=4444 > seleniumdriver.log 2>&1 &

      - name: Start DexIDP container
        run:
          docker run -d -p 5556:5556 -d -v $PWD/tools/dex.config.yaml:/etc/dex/config.docker.yaml --name dex ghcr.io/dexidp/dex:latest

      - name: Run keycloak tests
        working-directory: "tests/federation"
        env:
          KEYCLOAK_USER: admin
          KEYCLOAK_PASSWORD: password
        run: cargo test --test keycloak

      - name: Run dex tests
        working-directory: "tests/federation"
        env:
          DEX_URL: http://localhost:5556
        run: cargo test --test dex

      - name: Dump seleniumdriver log
        if: failure()
        run: cat seleniumdriver.log

      - name: Dump py-keystone logs
        if: failure()
        run: docker logs keystone

      - name: Dump rust keystone log
        if: failure()
        run: cat rust.log

      - name: Dump dex log
        if: failure()
        run: docker logs dex

      - name: Dump OPA log
        if: failure()
        run: docker logs opa

  federation-github:
    runs-on: ubuntu-latest
    if: "github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'openstack-experimental-release-plz'"
    needs:
      - build
    permissions:
      id-token: write
      contents: read
      packages: read
    services:
      postgres:
        image: postgres:17
        env:
          POSTGRES_USER: keystone
          POSTGRES_PASSWORD: '1234'
        ports:
          - 5432:5432
        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
      keycloak:
        image: ghcr.io/openstack-experimental/keystone/keycloak-ci-service:26.2
        env:
          KC_BOOTSTRAP_ADMIN_USERNAME: admin
          KC_BOOTSTRAP_ADMIN_PASSWORD: password
        ports:
          - 8082:8080
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - uses: ./.github/actions/deploy_keystone

      - name: Get GitHub JWT token
        id: get_token
        run: |
          TOKEN_JSON=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
          "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=https://github.com")

          TOKEN=$(echo $TOKEN_JSON | jq -r .value)
          echo "token=$TOKEN" >> $GITHUB_OUTPUT

      - name: Run github tests
        working-directory: "tests/federation"
        env:
          GITHUB_JWT: ${{ steps.get_token.outputs.token }}
          GITHUB_SUB: "repo:openstack-experimental/keystone:pull_request"
        run: cargo test --test github -- --nocapture

      - name: Dump py-keystone logs
        if: failure()
        run: docker logs keystone

      - name: Dump rust keystone log
        if: failure()
        run: cat rust.log

      - name: Dump OPA log
        if: failure()
        run: docker logs opa

  loadtest:
    runs-on: ubuntu-latest
    if: "github.actor != 'openstack-experimental-release-plz'"
    needs:
      - build
    permissions:
      contents: read
      packages: read
    services:
      postgres:
        image: postgres:17
        env:
          POSTGRES_USER: keystone
          POSTGRES_PASSWORD: '1234'
        ports:
          - 5432:5432
        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - uses: ./.github/actions/deploy_keystone

      - name: Build Load test binary
        working-directory: tests/loadtest
        run: cargo build --release

      - name: Run load test
        working-directory: tests/loadtest
        env:
          OS_CLOUD: admin
        run: |
          mkdir -p reports
          ./target/release/load_test \
            --host http://localhost:8080 \
            --hatch-rate 2 \
            --run-time 30s \
            --report-file reports/loadtest-report-rust.html \
            --report-file reports/loadtest-report-rust.md
          ./target/release/load_test \
            --host http://localhost:5001 \
            --hatch-rate 2 \
            --run-time 30s \
            --report-file reports/loadtest-report-python.html \
            --report-file reports/loadtest-report-python.md

      - name: Upload report
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: loadtest-report
          path: tests/loadtest/reports/

      - name: Dump py-keystone logs
        if: failure()
        run: docker logs keystone

      - name: Dump rust keystone log
        if: failure()
        run: cat rust.log

      - name: Dump OPA log
        if: failure()
        run: docker logs opa

  loadtest-track:
    if: "github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'openstack-experimental-release-plz'"
    runs-on: ubuntu-latest
    needs:
      - loadtest
    permissions:
      pull-requests: write
    steps:
      - name: Fetch pre-built artifacts
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v5.0.0
        with:
          name: loadtest-report

      - name: Extract report
        id: metrics
        run: |
          SUMMARY=$(cat loadtest-report-rust.md || echo "No summary found")
          echo "summary<<EOF" >> $GITHUB_OUTPUT
          echo "$SUMMARY" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: Post Loadtest results to PR
        if: github.event_name == 'pull_request'
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          header: loadtest
          message: |
            🦢 **Load Test Results**

            ${{ steps.metrics.outputs.summary }}

            [View full report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})

  k3s-test:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write # Required to push to GHCR
    steps:

      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1

      - name: Install K3s
        run: |
          # Install k3s without the heavy default components (Traefik/ServiceLB)
          curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik --disable servicelb --write-kubeconfig-mode 644" sh -s -

          # Export KUBECONFIG for the next steps
          echo "KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> $GITHUB_ENV

      - name: Install Skaffold
        run: |
          curl -Lo skaffold https://storage.googleapis.com/skaffold/builds/latest/skaffold-linux-amd64 && \
          sudo install skaffold /usr/local/bin/

      - name: Log in to GHCR
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2  # v4.0.0
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
        with:
          driver: docker-container

      - name: Wait for Cluster Ready
        run: |
          # Wait up to 60 seconds for the node to be ready
          timeout 60s bash -c 'until kubectl get nodes | grep -w "Ready"; do sleep 2; done'
          kubectl get nodes

      - name: Create GHCR Pull Secret
        run: |
          # Use --dry-run and pipe to 'apply' to make this step idempotent
          kubectl create secret docker-registry ghcr-pull-secret \
            --docker-server=ghcr.io \
            --docker-username=${{ github.actor }} \
            --docker-password=${{ secrets.GITHUB_TOKEN }} \
            --namespace=default \
            --dry-run=client -o yaml | kubectl apply -f -

      - name: Patch ServiceAccount for Automatic Pulling
        run: |
          # This ensures all pods in the namespace can pull from GHCR 
          # without you needing to edit every deployment.yaml
          kubectl patch serviceaccount default -p '{"imagePullSecrets": [{"name": "ghcr-pull-secret"}]}'

      - name: Deploy the development stack
        env:
          # This tells the underlying Docker Buildx to use GHA for cache
          # mode=max ensures all intermediate layers (like Cargo dependencies) are saved
          BUILDKIT_STEP_LOG_MAX_SIZE: 10485760
        run: |
          export BUILDX_NO_DEFAULT_ATTESTATIONS=1
          # first build artifacts preparing metadata file
          skaffold build --profile github-ci --default-repo ghcr.io/openstack-experimental/keystone --file-output build.artifacts
          # deploy built artifacts
          skaffold deploy -a build.artifacts --status-check

      - name: Run tests
        run: |
          # run tests referring to the artifacts metadata file
          skaffold verify -a build.artifacts -v debug

      - name: Dump K8 pods
        if: failure()
        run: kubectl get pods

      - name: Dump K8 pods - extended
        if: failure()
        run: kubectl get pods -o custom-columns="NAMESPACE:.metadata.namespace,POD:.metadata.name,STATUS:.status.phase,CONTAINER:.spec.containers[*].name,IMAGE:.spec.containers[*].image"

      - name: Dump skaffold images
        if: failure()
        run: cat build.artifacts

      - name: Dump keystone-rs logs
        if: failure()
        run: kubectl logs deployment/keystone-rs -c keystone