From eb605dc7a79391a5c504efa61b1ca6e74324e094 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 9 May 2025 18:28:03 +0000 Subject: [PATCH 1/7] copier --- .copier-answers.yml | 2 +- .github/workflows/ci.yaml | 44 +-------------- .github/workflows/pre-commit.yaml | 65 ++++++++++++++++++++++ template/.github/workflows/pre-commit.yaml | 65 ++++++++++++++++++++++ 4 files changed, 134 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 template/.github/workflows/pre-commit.yaml diff --git a/.copier-answers.yml b/.copier-answers.yml index a8080e13..e50e2452 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.42 +_commit: v0.0.42-3-g7069433 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9cc6c816..8685e7e7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,47 +21,9 @@ jobs: pre-commit: needs: [ get-values ] - strategy: - fail-fast: false - matrix: - os: - - "ubuntu-24.04" - python-version: - - 3.12.7 - name: Pre-commit for Py${{ matrix.python-version }} on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v4.2.2 - with: - ref: ${{ github.ref_name }} # explicitly get the head of - - - - - name: Install latest versions of python packages - uses: ./.github/actions/install_deps_uv - with: - python-version: ${{ matrix.python-version }} - - - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 - if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 - uses: ben-z/gh-action-mutex@1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10 - with: - branch: mutex-venv-${{ matrix.os }}-${{ matrix.python-version }} - timeout-minutes: 30 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it - - - name: Cache Pre-commit hooks - uses: actions/cache@v4.2.2 - env: - cache-name: cache-pre-commit-hooks - with: - path: ${{ env.PRE_COMMIT_HOME }} - key: ${{ matrix.os }}-${{ matrix.python-version }}-build-${{ env.cache-name }}-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: | - ${{ matrix.os }}-${{ matrix.python-version }}-build-${{ env.cache-name }}- - - - name: Run pre-commit - run: pre-commit run -a + uses: ./.github/workflows/pre-commit.yaml + with: + python-version: 3.12.7 lint-matrix: needs: [ pre-commit ] diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 00000000..3a98c839 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,65 @@ +name: Pre-commit + +on: + workflow_call: + inputs: + python-version: + description: 'What version of python' + type: string + required: true + setup-node: + description: 'Whether to set up Node' + type: boolean + default: false + node-version: + description: 'What version of node' + type: string + required: false + default: 'notUsing' + +env: + PYTHONUNBUFFERED: True + +permissions: + contents: write # needed for mutex + +jobs: + pre-commit: + runs-on: ubuntu-24.04 + name: Pre-commit + steps: + - name: Checkout code + uses: actions/checkout@v4.2.2 + with: + ref: ${{ github.ref_name }} # explicitly get the head of + + - name: Setup node + uses: actions/setup-node@v4.3.0 + if: ${{ inputs.setup-node }} + with: + node-version: ${{ inputs.node-version }} + + - name: Install latest versions of python packages + uses: ./.github/actions/install_deps_uv + with: + python-version: ${{ inputs.python-version }} + + - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 + if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 + uses: ben-z/gh-action-mutex@1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10 + with: + branch: mutex-venv-ubuntu-24.04-py${{ inputs.python-version }}-nodejs-${{ inputs.node-version }} + timeout-minutes: 30 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it + + - name: Cache Pre-commit hooks + uses: actions/cache@v4.2.2 + env: + cache-name: cache-pre-commit-hooks + with: + path: ${{ env.PRE_COMMIT_HOME }} + key: ubuntu-24.04-py${{ inputs.python-version }}-node-${{ inputs.node-version}}-${{ env.cache-name }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + ubuntu-24.04-py${{ inputs.python-version }}-node-${{ inputs.node-version}}-${{ env.cache-name }}- + + - name: Run pre-commit + run: pre-commit run -a diff --git a/template/.github/workflows/pre-commit.yaml b/template/.github/workflows/pre-commit.yaml new file mode 100644 index 00000000..3a98c839 --- /dev/null +++ b/template/.github/workflows/pre-commit.yaml @@ -0,0 +1,65 @@ +name: Pre-commit + +on: + workflow_call: + inputs: + python-version: + description: 'What version of python' + type: string + required: true + setup-node: + description: 'Whether to set up Node' + type: boolean + default: false + node-version: + description: 'What version of node' + type: string + required: false + default: 'notUsing' + +env: + PYTHONUNBUFFERED: True + +permissions: + contents: write # needed for mutex + +jobs: + pre-commit: + runs-on: ubuntu-24.04 + name: Pre-commit + steps: + - name: Checkout code + uses: actions/checkout@v4.2.2 + with: + ref: ${{ github.ref_name }} # explicitly get the head of + + - name: Setup node + uses: actions/setup-node@v4.3.0 + if: ${{ inputs.setup-node }} + with: + node-version: ${{ inputs.node-version }} + + - name: Install latest versions of python packages + uses: ./.github/actions/install_deps_uv + with: + python-version: ${{ inputs.python-version }} + + - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 + if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 + uses: ben-z/gh-action-mutex@1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10 + with: + branch: mutex-venv-ubuntu-24.04-py${{ inputs.python-version }}-nodejs-${{ inputs.node-version }} + timeout-minutes: 30 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it + + - name: Cache Pre-commit hooks + uses: actions/cache@v4.2.2 + env: + cache-name: cache-pre-commit-hooks + with: + path: ${{ env.PRE_COMMIT_HOME }} + key: ubuntu-24.04-py${{ inputs.python-version }}-node-${{ inputs.node-version}}-${{ env.cache-name }}-${{ hashFiles('.pre-commit-config.yaml') }} + restore-keys: | + ubuntu-24.04-py${{ inputs.python-version }}-node-${{ inputs.node-version}}-${{ env.cache-name }}- + + - name: Run pre-commit + run: pre-commit run -a From 39d2e75c731d04d17f5c00f44d6969ab7e6fc7df Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 9 May 2025 18:31:27 +0000 Subject: [PATCH 2/7] fix --- .copier-answers.yml | 2 +- .github/workflows/pre-commit.yaml | 3 ++- template/.github/workflows/pre-commit.yaml | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index e50e2452..a96d67cb 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.42-3-g7069433 +_commit: v0.0.42-5-g37a08e8 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 3a98c839..beaaadbd 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -19,6 +19,7 @@ on: env: PYTHONUNBUFFERED: True + PRE_COMMIT_HOME: ${{ github.workspace }}/.precommit_cache permissions: contents: write # needed for mutex @@ -31,7 +32,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4.2.2 with: - ref: ${{ github.ref_name }} # explicitly get the head of + ref: ${{ github.ref_name }} # explicitly get the head of the branch, which will include any new commits pushed if this is a dependabot branch - name: Setup node uses: actions/setup-node@v4.3.0 diff --git a/template/.github/workflows/pre-commit.yaml b/template/.github/workflows/pre-commit.yaml index 3a98c839..beaaadbd 100644 --- a/template/.github/workflows/pre-commit.yaml +++ b/template/.github/workflows/pre-commit.yaml @@ -19,6 +19,7 @@ on: env: PYTHONUNBUFFERED: True + PRE_COMMIT_HOME: ${{ github.workspace }}/.precommit_cache permissions: contents: write # needed for mutex @@ -31,7 +32,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4.2.2 with: - ref: ${{ github.ref_name }} # explicitly get the head of + ref: ${{ github.ref_name }} # explicitly get the head of the branch, which will include any new commits pushed if this is a dependabot branch - name: Setup node uses: actions/setup-node@v4.3.0 From 5fe9b4d1bb5fe4821fa1b92653f2865d6e0df47e Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 9 May 2025 18:32:27 +0000 Subject: [PATCH 3/7] use in template --- template/.github/workflows/ci.yaml.jinja | 35 ++---------------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/template/.github/workflows/ci.yaml.jinja b/template/.github/workflows/ci.yaml.jinja index 73c51acb..50e8fbcc 100644 --- a/template/.github/workflows/ci.yaml.jinja +++ b/template/.github/workflows/ci.yaml.jinja @@ -22,38 +22,9 @@ jobs: lint: needs: [ get-values ] name: Pre-commit - runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} - steps: - - name: Checkout code - uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} - - - name: Install latest versions of python packages - uses: ./.github/actions/install_deps_uv - with: - python-version: {% endraw %}{{ python_version }}{% raw %}{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} - code-artifact-auth-role-name: CoreInfraBaseAccess - code-artifact-auth-role-account-id: {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} - code-artifact-auth-region: {% endraw %}{{ aws_org_home_region }}{% endif %}{% raw %} - - - name: Set up mutex # Github concurrency management is horrible, things get arbitrarily cancelled if queued up. So using mutex until github fixes itself. When multiple jobs are modifying cache at once, weird things can happen. possible issue is https://github.com/actions/toolkit/issues/658 - if: ${{ runner.os != 'Windows' }} # we're just gonna have to YOLO on Windows, because this action doesn't support it yet https://github.com/ben-z/gh-action-mutex/issues/14 - uses: ben-z/gh-action-mutex@{% endraw %}{{ gha_mutex }}{% raw %} - with: - branch: mutex-venv-{% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %} - timeout-minutes: 30 # this is the amount of time this action will wait to attempt to acquire the mutex lock before failing, e.g. if other jobs are queued up in front of it - - - name: Cache Pre-commit hooks - uses: actions/cache@{% endraw %}{{ gha_cache }}{% raw %} - env: - cache-name: cache-pre-commit-hooks - with: - path: ${{ env.PRE_COMMIT_HOME }} - key: {% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %}-build-${{ env.cache-name }}-${{ hashFiles('.pre-commit-config.yaml') }} - restore-keys: | - {% endraw %}{{ gha_linux_runner }}{% raw %}-py{% endraw %}{{ python_version }}{% raw %}-build-${{ env.cache-name }}- - - - name: Run pre-commit - run: pre-commit run -a + uses: ./.github/workflows/pre-commit.yaml + with: + python-version: {% endraw %}{{ python_version }}{% raw %} test: needs: [ lint ] From 972bfffb13e21f6c9aa4da4ad045fc720b99e03f Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 9 May 2025 18:35:01 +0000 Subject: [PATCH 4/7] copier --- .copier-answers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index a96d67cb..c710baeb 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.42-5-g37a08e8 +_commit: v0.0.42-6-g3baae14 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: From 49cfac2ff0e3fad4d882fa6a2e19a410617a1211 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 9 May 2025 18:36:24 +0000 Subject: [PATCH 5/7] add parent --- .github/dependabot.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ec71f925..6dd52fea 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,7 +12,10 @@ updates: - dependency-name: "boto3" # boto3 gets patch updates way too frequently and they're usually not important update-types: - "version-update:semver-patch" - + - dependency-name: "sphinx*" # read-the-docs uses specific versions of sphinx, so we generally want to stay tightly pinned unless there's a major version change + update-types: + - "version-update:semver-minor" + - "version-update:semver-patch" groups: prod-dependencies: dependency-type: "production" From 09c90e96a87aecc36e3fe3e87e40361ddd707a25 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 9 May 2025 18:40:47 +0000 Subject: [PATCH 6/7] copier --- .copier-answers.yml | 2 +- .github/dependabot.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index c710baeb..78a86af6 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.42-6-g3baae14 +_commit: v0.0.43-1-gfa4f806 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6dd52fea..7a7f4a7b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,7 @@ updates: update-types: - "version-update:semver-minor" - "version-update:semver-patch" + groups: prod-dependencies: dependency-type: "production" From 8bca3fdd438baab9e4b66142b3060375d191041a Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 9 May 2025 18:41:45 +0000 Subject: [PATCH 7/7] tag --- .copier-answers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 78a86af6..ee95a918 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.43-1-gfa4f806 +_commit: v0.0.44 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: