copier-python-package-template/.github/workflows/ci.yaml at 7dcb2dbbb5a78d07dd3f2159454a647be6401630 · LabAutomationAndScreening/copier-python-package-template · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
name: CI

on:
  push:
    branches-ignore:
      - 'gh-readonly-queue/**' # don't run (again) when on these special branches created during merge groups; the `on: merge_group` already triggers it.
  merge_group:

env:
  PYTHONUNBUFFERED: True
  PRE_COMMIT_HOME: ${{ github.workspace }}/.precommit_cache

permissions:
    id-token: write
    contents: write # needed for mutex

jobs:
  pre-commit:
    strategy:
      fail-fast: false
      matrix:
        os:
          - "ubuntu-24.04"
        python-version:
            - 3.12.7
    name: Pre-commit for Py${{ matrix.python-version }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4.2.2


      - name: Install latest versions of python packages
        uses: ./.github/actions/install_deps_uv
        with:
          python-version: ${{ matrix.python-version }}

      - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen.  possible issue is https://github.com/actions/toolkit/issues/658
        if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14
        uses: ben-z/gh-action-mutex@1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10
        with:
          branch: mutex-venv-${{ matrix.os }}-${{ matrix.python-version }}
        timeout-minutes: 30 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it

      - name: Cache Pre-commit hooks
        uses: actions/cache@v4.2.2
        env:
          cache-name: cache-pre-commit-hooks
        with:
          path: ${{ env.PRE_COMMIT_HOME }}
          key: ${{ matrix.os }}-${{ matrix.python-version }}-build-${{ env.cache-name }}-${{ hashFiles('.pre-commit-config.yaml') }}
          restore-keys: |
            ${{ matrix.os }}-${{ matrix.python-version }}-build-${{ env.cache-name }}-

      - name: Run pre-commit
        run:  pre-commit run -a

  lint-matrix:
    needs: [ pre-commit ]
    strategy:
      matrix:
        os:
          - "ubuntu-24.04"
        python-version:

          - 3.12.7

          - 3.13.2

        copier: [
          '--data-file tests/copier_data/data1.yaml',
          '--data-file tests/copier_data/data2.yaml',
          '--data-file tests/copier_data/data3.yaml',
          ]
    runs-on: ${{ matrix.os }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4.2.2

      - name: Move python script that replaces private package registry information to temp folder so it doesn't get deleted
        run: |
          mv .github/workflows/replace_private_package_registries.py $RUNNER_TEMP

      - name: Install python tooling
        uses: ./.github/actions/install_deps_uv
        with:
          python-version: ${{ matrix.python-version }}
          uv-sync: false

      - name: Instantiate copier template
        run: |
          copier copy --trust --vcs-ref ${{ github.sha }} ${{ matrix.copier }} --data python_version=${{ matrix.python-version }} . ./new-template
      - name: Delete files from initial repo
        run: |
          # Delete everything except the folder containing the instantiated template
          # https://stackoverflow.com/questions/34057047/delete-all-directories-except-one
          shopt -s extglob
          rm -rf !(new-template)
          rm -rf .github # apparently this folder doesn't get removed with the previous command for some reason
          rm -rf .devcontainer # apparently this folder doesn't get removed with the previous command for some reason
          ls -la
      - name: Move the instantiated template into the repo root
        run: |
          # Move all the files from the instantiated template out of the subfolder
          shopt -s dotglob # https://unix.stackexchange.com/questions/6393/how-do-you-move-all-files-including-hidden-from-one-directory-to-another
          mv new-template/* .
          ls -la
          # delete the subfolder
          rm -frd new-template
          ls -la


      - name: install new dependencies
        env:
          # Provide a fake token so it doesn't attempt to access AWS to generate a new one when the script is run if CodeArtifact is set as the registry
          CODEARTIFACT_AUTH_TOKEN: 'faketoken'
        run: |
          # Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI
          python $RUNNER_TEMP/replace_private_package_registries.py
          sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock
          # Add everything to git so that pre-commit recognizes the files and runs on them
          git add .
          git status

      - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen.  possible issue is https://github.com/actions/toolkit/issues/658
        if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14
        uses: ben-z/gh-action-mutex@1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10
        with:
          branch: mutex-venv-${{ matrix.os }}-${{ matrix.python-version }}
        timeout-minutes: 30 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it

      - name: Cache Pre-commit hooks
        uses: actions/cache@v4.2.2
        env:
          cache-name: cache-pre-commit-hooks
        with:
          path: ${{ env.PRE_COMMIT_HOME }}
          key: ${{ matrix.os }}-${{ matrix.python-version }}-build-${{ env.cache-name }}-${{ hashFiles('.pre-commit-config.yaml') }}
          restore-keys: |
            ${{ matrix.os }}-${{ matrix.python-version }}-build-${{ env.cache-name }}-

      - name: Run pre-commit
        run: |
          # skip devcontainer context hash because the template instantiation may make it different every time
          SKIP=git-dirty,compute-devcontainer-context-hash pre-commit run -a

      - name: Upload pre-commit log if failure
        if: ${{ failure() }}
        uses: actions/upload-artifact@v4.4.3
        with:
          name: pre-commit-log--${{ github.jobs.lint-matrix.name }}
          path: "${{ github.workspace }}/.precommit_cache/pre-commit.log"

  required-check:
    runs-on: ubuntu-24.04
    needs: [ lint-matrix ]
    if: always()
    steps:
      - name: fail if prior job failure
        if: needs.lint-matrix.result != 'success'
        run: |
          exit 1