test.yml

---
name: Tests

on:
  workflow_dispatch:

  push:
    branches:
      - main
      - develop
    tags:
      - "v*.*.*"

  pull_request:
    branches:
      - main
      - develop

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  test:
    name: (${{ matrix.os }},Py${{ matrix.python-version }},sk${{ matrix.scikit-learn }}${{ matrix.pandas-version != '' && format(',pd:{0}', matrix.pandas-version) || '' }},sk-only:${{ matrix.sklearn-only }})
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
        scikit-learn: ["1.3.*", "1.4.*", "1.5.*", "1.6.*", "1.7.*"]
        os: [ubuntu-latest]
        sklearn-only: ["true"]

        exclude:
          # incompatible version combinations
          - python-version: "3.13"
            scikit-learn: "1.3.*"
          - python-version: "3.13"
            scikit-learn: "1.4.*"
          - python-version: "3.14"
            scikit-learn: "1.3.*"
          - python-version: "3.14"
            scikit-learn: "1.4.*"

        include:
          # Full test run on ubuntu, 3.14
          - os: ubuntu-latest
            python-version: "3.14"
            scikit-learn: "1.7.*"
            sklearn-only: "false"

          # Full test run on Windows
          - os: windows-latest
            python-version: "3.12"
            scikit-learn: "1.5.*"
            sklearn-only: "false"

          # Coverage run
          - os: ubuntu-latest
            python-version: "3.12"
            scikit-learn: "1.5.*"
            sklearn-only: "false"
            code-cov: true

          # Pandas 2 run
          - os: ubuntu-latest
            python-version: "3.12"
            scikit-learn: "1.5.*"
            sklearn-only: "false"
            pandas-version: "2.*"
            code-cov: false

    steps:
    - uses: actions/checkout@v6
      with:
        fetch-depth: 2

    - name: Setup Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install test dependencies, scikit-learn, and optional pandas
      shell: bash
      run: |
        python -m pip install --upgrade pip
        pip install -e .[test] scikit-learn==${{ matrix.scikit-learn }}
        
        if [ "${{ matrix.pandas-version }}" != "" ]; then
          echo "Installing specific pandas version: ${{ matrix.pandas-version }}"
          pip install "pandas==${{ matrix.pandas-version }}"
        fi

    - name: Store repository status
      id: status-before
      if: matrix.os != 'windows-latest'
      run: |
        git_status=$(git status --porcelain -b)
        echo "BEFORE=$git_status" >> $GITHUB_ENV
        echo "Repository status before tests: $git_status"

    - name: Show installed dependencies
      run: python -m pip list

    - name: Run tests on Ubuntu Test
      if: matrix.os == 'ubuntu-latest'
      env:
        OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
      run: |
        if [ "${{ matrix.code-cov }}" = "true" ]; then
          codecov="--cov=openml --long --cov-report=xml"
        fi

        if [ "${{ matrix.sklearn-only }}" = "true" ]; then
          marks="sklearn and not production and not uses_test_server"
        else
          marks="not production and not uses_test_server"
        fi

        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

    - name: Run tests on Ubuntu Production
      if: matrix.os == 'ubuntu-latest'
      env:
        OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
      run: |
        if [ "${{ matrix.code-cov }}" = "true" ]; then
          codecov="--cov=openml --long --cov-report=xml"
        fi

        if [ "${{ matrix.sklearn-only }}" = "true" ]; then
          marks="sklearn and production and not uses_test_server"
        else
          marks="production and not uses_test_server"
        fi

        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

    - name: Run tests on Windows
      if: matrix.os == 'windows-latest'
      env:
        OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
      run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
        pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not uses_test_server"

    - name: Check for files left behind by test
      if: matrix.os != 'windows-latest' && always()
      run: |
        before="${{ env.BEFORE }}"
        after="$(git status --porcelain -b)"
        if [[ "$before" != "$after" ]]; then
            echo "git status from before: $before"
            echo "git status from after: $after"
            echo "Not all generated files have been deleted!"
            exit 1
        fi

    - name: Upload coverage
      if: matrix.code-cov && always()
      uses: codecov/codecov-action@v4
      with:
        files: coverage.xml
        token: ${{ secrets.CODECOV_TOKEN }}
        fail_ci_if_error: true
        verbose: true

  dummy_windows_py_sk024:
    name: (windows-latest, Py, sk0.24.*, sk-only:false)
    runs-on: ubuntu-latest
    steps:
      - name: Dummy step
        run: |
          echo "This is a temporary dummy job."
          echo "Always succeeds."

  dummy_windows_py_sk023:
    name: (ubuntu-latest, Py3.8, sk0.23.1, sk-only:false)
    runs-on: ubuntu-latest
    steps:
      - name: Dummy step
        run: |
          echo "This is a temporary dummy job."
          echo "Always succeeds."

  dummy_docker:
    name: docker
    runs-on: ubuntu-latest
    steps:
      - name: Dummy step
        run: |
          echo "This is a temporary dummy docker job."
          echo "Always succeeds."