integration-test.yml

# This workflow will build a golang project
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go

name: Docker Integration Test

on:
  push:
    branches:
      - main
      - release/**
  pull_request:
    branches:
      - main
      - release/**

defaults:
  run:
    shell: bash

jobs:
  integration-tests:
    name: Integration Test (${{ matrix.test.name }})
    runs-on: ubuntu-large
    timeout-minutes: 30
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      DAPP_TESTS_MNEMONIC: ${{ secrets.DAPP_TESTS_MNEMONIC }}
    strategy:
      # other jobs should run even if one integration test fails
      fail-fast: false
      matrix:
        test: [
          {
            name: "Wasm Module",
            scripts: [
              "docker exec sei-node-0 integration_test/contracts/deploy_timelocked_token_contract.sh",
              "python3 integration_test/scripts/runner.py integration_test/wasm_module/timelocked_token_delegation_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/wasm_module/timelocked_token_admin_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/wasm_module/timelocked_token_withdraw_test.yaml",
              "docker exec sei-node-0 integration_test/contracts/deploy_timelocked_token_contract.sh",
              "python3 integration_test/scripts/runner.py integration_test/wasm_module/timelocked_token_emergency_withdraw_test.yaml"
            ]
          },
          {
            name: "Mint & Staking & Bank Module",
            scripts: [
              "python3 integration_test/scripts/runner.py integration_test/staking_module/staking_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/bank_module/send_funds_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/mint_module/mint_test.yaml"
            ]
          },
          {
            name: "Gov & Oracle & Authz Module",
            scripts: [
              "python3 integration_test/scripts/runner.py integration_test/gov_module/gov_proposal_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/gov_module/staking_proposal_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/oracle_module/verify_penalty_counts.yaml",
              "python3 integration_test/scripts/runner.py integration_test/oracle_module/set_feeder_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/authz_module/send_authorization_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/authz_module/staking_authorization_test.yaml",
              "python3 integration_test/scripts/runner.py integration_test/authz_module/generic_authorization_test.yaml"
            ]
          },
          {
            name: "Chain Operation Test",
            scripts: [
              "until [ $(cat build/generated/rpc-launch.complete |wc -l) = 1 ]; do sleep 10; done",
              "until [[ $(docker exec sei-rpc-node build/seid status |jq -M -r .SyncInfo.latest_block_height) -gt 10 ]]; do sleep 10; done",
              "echo rpc node started",
              "python3 integration_test/scripts/runner.py integration_test/chain_operation/snapshot_operation.yaml",
              "python3 integration_test/scripts/runner.py integration_test/chain_operation/statesync_operation.yaml"
            ]
          },
          {
            name: "Distribution Module",
            scripts: [
              "python3 integration_test/scripts/runner.py integration_test/distribution_module/community_pool.yaml",
              "python3 integration_test/scripts/runner.py integration_test/distribution_module/rewards.yaml",
            ]
          },
          {
            name: "Upgrade Module (Major)",
            env: "UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2",
            scripts: [
              "python3 integration_test/scripts/runner.py integration_test/upgrade_module/major_upgrade_test.yaml"
            ]
          },
          {
            name: "Upgrade Module (Minor)",
            env: "UPGRADE_VERSION_LIST=v1.0.0,v1.0.1,v1.0.2",
            scripts: [
              "python3 integration_test/scripts/runner.py integration_test/upgrade_module/minor_upgrade_test.yaml"
            ]
          },
          {
            name: "SeiDB State Store",
            scripts: [
              "docker exec sei-node-0 integration_test/contracts/deploy_wasm_contracts.sh",
              "docker exec sei-node-0 integration_test/contracts/create_tokenfactory_denoms.sh",
              "python3 integration_test/scripts/runner.py integration_test/seidb/state_store_test.yaml",
            ],
          },
          {
            name: "EVM Module",
            scripts: [
              "./integration_test/evm_module/scripts/evm_tests.sh",
            ]
          },
          {
            name: "EVM Module (Parquet)",
            env: "RECEIPT_STORE_BACKEND=parquet",
            scripts: [
              "./integration_test/evm_module/scripts/evm_tests.sh",
            ]
          },
          {
            name: "EVM Interoperability",
            scripts: [
              "./integration_test/evm_module/scripts/evm_interoperability_tests.sh"
            ]
          },
          {
            name: "Disable WASM Tests",
            scripts: [
              "./integration_test/evm_module/scripts/disable_wasm.sh",
            ]
          },
          {
            name: "dApp Tests",
            scripts: [
              "./integration_test/dapp_tests/dapp_tests.sh seilocal"
            ]
          },
          {
            name: "EVM GIGA Module",
            env: "GIGA_EXECUTOR=true GIGA_OCC=true",
            scripts: [
              "./integration_test/evm_module/scripts/evm_giga_tests.sh"
            ]
          },
          {
            name: "EVM GIGA Mixed (Determinism)",
            scripts: [
              "./integration_test/evm_module/scripts/evm_giga_mixed_tests.sh"
            ]
          },
          {
            name: "EVM RPC .io/.iox (spec fixtures)",
            scripts: [
              "./integration_test/evm_module/scripts/evm_rpc_tests.sh"
            ]
          },
        ]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - uses: actions/setup-node@v2
        with:
          node-version: '20'

      - name: Pyyaml
        run: |
          pip3 install pyyaml

      - name: Install jq
        run: sudo apt-get install -y jq

      - name: Set up Go
        uses: actions/setup-go@v3
        with:
          go-version: '1.25.6'
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        if: env.DOCKERHUB_USERNAME != ''
        env:
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Start 4 node docker cluster
        run: make clean && DOCKER_DETACH=true INVARIANT_CHECK_INTERVAL=10 ${{matrix.test.env}} make docker-cluster-start

      - name: Wait for docker cluster to start
        run: |
          until [ $(cat build/generated/launch.complete |wc -l) = 4 ]
          do
            sleep 10
          done
          sleep 10

      - name: Start rpc node
        run: make run-rpc-node-skipbuild &

      - name: Verify Sei Chain is running
        run: python3 integration_test/scripts/runner.py integration_test/startup/startup_test.yaml

      # EVM execution-apis .io is allowed to fail until endpoint coverage is fixed (non-blocking for PR merge)
      - name: ${{ matrix.test.name }}
        continue-on-error: ${{ matrix.test.name == 'EVM RPC .io/.iox (spec fixtures)' }}
        run: |
          scripts=$(echo '${{ toJson(matrix.test.scripts) }}' | jq -r '.[]')
          IFS=$'\n'   # change the internal field separator to newline
          echo $scripts
          for script in $scripts
          do
            bash -c "${script}"
          done
          unset IFS  # revert the internal field separator back to default

      - name: Print node logs on failure
        if: ${{ failure() }}
        run: |
          set -euo pipefail
          for c in sei-node-0 sei-node-1 sei-node-2 sei-node-3; do
            echo "==================== ${c} (docker logs tail) ===================="
            docker logs --tail 200 "${c}" || true
            echo "==================== ${c} (seid log file tail) ===================="
            # Logs are accessible on host since build/generated is mounted in containers
            NODE_ID=${c#sei-node-}
            if [ -f "build/generated/logs/seid-${NODE_ID}.log" ]; then
              tail -200 "build/generated/logs/seid-${NODE_ID}.log" || true
            fi
          done

      - name: Prepare log artifact name
        if: ${{ always() }}
        id: log_artifact_meta
        run: |
          raw="${{ matrix.test.name }}"
          safe=$(echo "$raw" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g' | sed -E 's/^-+|-+$//g')
          if [ -z "$safe" ]; then
            safe="logs"
          fi
          echo "artifact_name=$safe" >> $GITHUB_OUTPUT

      - name: Collect logs directory
        if: ${{ always() }}
        run: |
          LOG_ROOT="artifacts/sei-${{ steps.log_artifact_meta.outputs.artifact_name }}"
          mkdir -p "$LOG_ROOT"
          if [ -d build/generated/logs ]; then
            cp -r build/generated/logs "$LOG_ROOT/"
          else
            echo "No logs directory found"
          fi

      - name: Upload logs directory
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: integration-logs-${{ steps.log_artifact_meta.outputs.artifact_name }}
          path: artifacts/sei-${{ steps.log_artifact_meta.outputs.artifact_name }}
          if-no-files-found: warn

  integration-test-check:
    name: Integration Test Check
    runs-on: ubuntu-latest
    needs: integration-tests
    if: always()
    steps:
      - name: Get workflow conclusion
        id: workflow_conclusion
        uses: nick-fields/retry@v2
        with:
          max_attempts: 2
          retry_on: error
          timeout_seconds: 30
          command: |
              jobs=$(curl https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs)
              job_statuses=$(echo "$jobs" | jq -r '.jobs[] | .conclusion')

              for status in $job_statuses
              do
                echo "Status: $status"
                if [[ "$status" == "failure" ]]; then
                  echo "Some or all tests have failed!"
                  exit 1
                fi
                if [[ "$status" == "cancelled" ]]; then
                  echo "Some or all tests have been cancelled!"
                  exit 1
                fi
              done

              echo "All tests have passed!"