From e0484dfca722fd65dd24b991d4049f84479e20e6 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 10:57:07 +0000 Subject: [PATCH 01/20] update publish job --- template/.github/workflows/publish.yaml.jinja | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 39c0ddda..79c339c8 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -109,6 +109,13 @@ jobs: {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} uv build --no-sources + - name: Upload build package + uses: actions/upload-artifact@v4.6.2 + with: + name: python-package-distributions + path: dist/ + if-no-files-found: error + {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} - name: OIDC Auth for Publishing to CodeArtifact uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} @@ -121,4 +128,25 @@ jobs: - name: Publish package run: | {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} - uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}pypi{% endif %} + uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}pypi{% endif %}{% raw %} + + publish: + name: Publish Python 🐍 distribution 📦 to PyPI + needs: [ build ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + environment: + name: pypi + url: https://pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} + permissions: + attestations: write + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@v1.12.4 + with: + attestations: false{% endraw %} From df1044f9859c9caf5531b811f868dec1c3188a9b Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 11:01:58 +0000 Subject: [PATCH 02/20] switch to test pypi --- template/.github/workflows/publish.yaml.jinja | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 79c339c8..396201c6 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -130,12 +130,12 @@ jobs: {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}pypi{% endif %}{% raw %} - publish: - name: Publish Python 🐍 distribution 📦 to PyPI + publish-to-staging: + name: Publish Python distribution to Staging Package Registry needs: [ build ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} environment: - name: pypi + name: testpypi url: https://pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} permissions: attestations: write @@ -149,4 +149,5 @@ jobs: - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@v1.12.4 with: - attestations: false{% endraw %} + attestations: false + repository-url: https://test.pypi.org/legacy/{% endraw %} From 626e2e5f9149d49b0ad58ef9477be62fc138a41b Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 11:05:08 +0000 Subject: [PATCH 03/20] remove ca --- template/.github/workflows/publish.yaml.jinja | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 396201c6..0c42e668 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -110,7 +110,7 @@ jobs: uv build --no-sources - name: Upload build package - uses: actions/upload-artifact@v4.6.2 + uses: actions/upload-artifact@{% endraw %}{{ gha_upload_artifact }}{% raw %} with: name: python-package-distributions path: dist/ @@ -125,10 +125,7 @@ jobs: {% endraw %}{% endif %}{% raw %} - - name: Publish package - run: | -{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} - uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}pypi{% endif %}{% raw %} + publish-to-staging: name: Publish Python distribution to Staging Package Registry @@ -142,7 +139,7 @@ jobs: id-token: write steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@{% endraw %}{{ gha_download_artifact }}{% raw %} with: name: python-package-distributions path: dist/ From 44c387977ca7f1c0315f5167b43060386afea5b4 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:23:13 +0000 Subject: [PATCH 04/20] git tag --- src/git_tag.py | 95 +++++++++++++++++++ template/.github/workflows/git_tag.py | 1 + template/.github/workflows/publish.yaml.jinja | 24 +++-- 3 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 src/git_tag.py create mode 120000 template/.github/workflows/git_tag.py diff --git a/src/git_tag.py b/src/git_tag.py new file mode 100644 index 00000000..e0bd5abf --- /dev/null +++ b/src/git_tag.py @@ -0,0 +1,95 @@ +import argparse +import subprocess +import tomllib +from pathlib import Path + + +def extract_version(toml_path: Path | str) -> str: + """Load toml_path and return the version string. + + Checks [project].version (PEP 621) first, then [tool.poetry].version. Raises KeyError if no version field is found. + """ + path = Path(toml_path) + with path.open("rb") as f: + data = tomllib.load(f) + + project = data.get("project", {}) + if version := project.get("version"): + return version + + tool = data.get("tool", {}) + poetry = tool.get("poetry", {}) + if version := poetry.get("version"): + return version + + raise KeyError(f"No version field found in {path!r}") # noqa: TRY003 # not worth a custom exception + + +def ensure_tag_not_present(tag: str, remote: str) -> None: + try: + _ = subprocess.run( # noqa: S603 # this is trusted input, it's our own arguments being passed in + ["git", "ls-remote", "--exit-code", "--tags", remote, f"refs/tags/{tag}"], # noqa: S607 # if `git` isn't in PATH already, then there are bigger problems to solve + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + raise Exception(f"Error: tag '{tag}' exists on remote '{remote}'") # noqa: TRY002,TRY003 # not worth a custom exception + except subprocess.CalledProcessError: + # tag not present, continue + return + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Extract the version from a pyproject.toml file, " + "confirm that git tag v is not present, or " + "create and push the tag to a remote." + ) + ) + _ = parser.add_argument( + "file", + nargs="?", + default="pyproject.toml", + help="Path to pyproject.toml (default: pyproject.toml)", + ) + _ = parser.add_argument( + "--confirm-tag-not-present", + action="store_true", + help=("Check that git tag v is NOT present on the remote. If the tag exists, exit with an error."), + ) + _ = parser.add_argument( + "--push-tag-to-remote", + action="store_true", + help=( + "Create git tag v locally and push it to the remote. " + "Internally confirms the tag is not already present." + ), + ) + _ = parser.add_argument( + "--remote", + default="origin", + help="Name of git remote to query/push (default: origin)", + ) + args = parser.parse_args() + + ver = extract_version(args.file) + + tag = f"v{ver}" + + if args.push_tag_to_remote: + ensure_tag_not_present(tag, args.remote) + _ = subprocess.run(["git", "tag", tag], check=True) # noqa: S603,S607 # this is trusted input, it's our own pyproject.toml file. and if `git` isn't in PATH, then there are larger problems anyway + _ = subprocess.run(["git", "push", args.remote, tag], check=True) # noqa: S603,S607 # this is trusted input, it's our own pyproject.toml file. and if `git` isn't in PATH, then there are larger problems anyway + return + + if args.confirm_tag_not_present: + ensure_tag_not_present(tag, args.remote) + return + + # Default behavior: just print the version + print(ver) # noqa: T201 # specifically printing this out so CI pipelines can read the value from stdout + + +if __name__ == "__main__": + main() diff --git a/template/.github/workflows/git_tag.py b/template/.github/workflows/git_tag.py new file mode 120000 index 00000000..3f6f11c3 --- /dev/null +++ b/template/.github/workflows/git_tag.py @@ -0,0 +1 @@ +../../../src/git_tag.py \ No newline at end of file diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 0c42e668..dd185e62 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -116,14 +116,7 @@ jobs: path: dist/ if-no-files-found: error -{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} - - name: OIDC Auth for Publishing to CodeArtifact - uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} - with: - role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} - aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} -{% endraw %}{% endif %}{% raw %} @@ -133,7 +126,7 @@ jobs: runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} environment: name: testpypi - url: https://pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} + url: https://test.pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} permissions: attestations: write id-token: write @@ -143,7 +136,20 @@ jobs: with: name: python-package-distributions path: dist/ - - name: Publish distribution 📦 to PyPI +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Publishing to CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + + - name: Publish distribution to Code Artifact + run: | + . .devcontainer/code-artifact-auth.sh + uv publish --verbose --index code-artifact-primary --username aws --password "$TWINE_PASSWORD" + +{% endraw %}{% endif %}{% raw %} + - name: Publish distribution to PyPI uses: pypa/gh-action-pypi-publish@v1.12.4 with: attestations: false From 5f5a5004e97a7828df5bd35804a486ead6f820df Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:36:20 +0000 Subject: [PATCH 05/20] test staging --- template/.github/workflows/publish.yaml.jinja | 50 +++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index dd185e62..e4704382 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -12,6 +12,21 @@ permissions: contents: write # needed for mutex jobs: + get-values: + name: Get Values + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + - name: Extract package version + run: | + VERSION=$(python3 ./github/workflows/git_tag.py) + echo "package_version=$VERSION" >> $GITHUB_OUTPUT + lint: name: Pre-commit runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} @@ -117,9 +132,6 @@ jobs: if-no-files-found: error - - - publish-to-staging: name: Publish Python distribution to Staging Package Registry needs: [ build ] @@ -153,4 +165,34 @@ jobs: uses: pypa/gh-action-pypi-publish@v1.12.4 with: attestations: false - repository-url: https://test.pypi.org/legacy/{% endraw %} + repository-url: https://test.pypi.org/legacy/ + + install-from-staging: + name: Install package from staging registry + needs: [ publish-to-staging, get-values ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + - name: Install from staging registry + run: pip install -i https://test.pypi.org/simple/ {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package_version }} + - name: Confirm library can be importde successfully + run: python -c "import {% endraw %}{{ package_name }}{% raw %}" + + create-tag: + name: Create the git tag + needs: [ install-from-staging ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + - name: Confirm tag not already present + run: python3 ./github/workflows/git_tag.py --confirm-tag-not-present + - name: Create tag + run: python3 ./github/workflows/git_tag.py --push-tag-to-remote{% endraw %} From 799ca6e4292deaaf65a0140778cbd94fce61a858 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:37:24 +0000 Subject: [PATCH 06/20] syntax --- template/.github/workflows/publish.yaml.jinja | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index e4704382..ce68b1e1 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -22,10 +22,10 @@ jobs: uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} with: python-version: {% endraw %}{{ python_version }}{% raw %} - - name: Extract package version - run: | - VERSION=$(python3 ./github/workflows/git_tag.py) - echo "package_version=$VERSION" >> $GITHUB_OUTPUT + - name: Extract package version + run: | + VERSION=$(python3 ./github/workflows/git_tag.py) + echo "package_version=$VERSION" >> $GITHUB_OUTPUT lint: name: Pre-commit From 8835e685f16ee008b897f10ddb2c340111aaa02e Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:38:18 +0000 Subject: [PATCH 07/20] whitespace --- template/.github/workflows/publish.yaml.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index ce68b1e1..1c404cc7 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -167,7 +167,7 @@ jobs: attestations: false repository-url: https://test.pypi.org/legacy/ - install-from-staging: + install-from-staging: name: Install package from staging registry needs: [ publish-to-staging, get-values ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} @@ -181,7 +181,7 @@ jobs: - name: Confirm library can be importde successfully run: python -c "import {% endraw %}{{ package_name }}{% raw %}" - create-tag: + create-tag: name: Create the git tag needs: [ install-from-staging ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} From 3b6b7f1a491e1ebeddfc768ae416409be9bf1337 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:40:13 +0000 Subject: [PATCH 08/20] github --- template/.github/workflows/publish.yaml.jinja | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 1c404cc7..643cc8df 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -24,7 +24,7 @@ jobs: python-version: {% endraw %}{{ python_version }}{% raw %} - name: Extract package version run: | - VERSION=$(python3 ./github/workflows/git_tag.py) + VERSION=$(python3 ./.github/workflows/git_tag.py) echo "package_version=$VERSION" >> $GITHUB_OUTPUT lint: @@ -193,6 +193,6 @@ jobs: with: python-version: {% endraw %}{{ python_version }}{% raw %} - name: Confirm tag not already present - run: python3 ./github/workflows/git_tag.py --confirm-tag-not-present + run: python3 ./.github/workflows/git_tag.py --confirm-tag-not-present - name: Create tag - run: python3 ./github/workflows/git_tag.py --push-tag-to-remote{% endraw %} + run: python3 ./.github/workflows/git_tag.py --push-tag-to-remote{% endraw %} From 4f7a6b35b06c3d1e882abd6649652a855f411df6 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:47:45 +0000 Subject: [PATCH 09/20] echo --- template/.github/workflows/publish.yaml.jinja | 58 +++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 643cc8df..7268829b 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -160,12 +160,14 @@ jobs: . .devcontainer/code-artifact-auth.sh uv publish --verbose --index code-artifact-primary --username aws --password "$TWINE_PASSWORD" -{% endraw %}{% endif %}{% raw %} - - name: Publish distribution to PyPI +{% endraw %}{% else %}{% raw %} + - name: Publish distribution to Test PyPI uses: pypa/gh-action-pypi-publish@v1.12.4 with: attestations: false repository-url: https://test.pypi.org/legacy/ +{% endraw %}{% endif %}{% raw %} + install-from-staging: name: Install package from staging registry @@ -178,7 +180,7 @@ jobs: python-version: {% endraw %}{{ python_version }}{% raw %} - name: Install from staging registry run: pip install -i https://test.pypi.org/simple/ {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package_version }} - - name: Confirm library can be importde successfully + - name: Confirm library can be imported successfully run: python -c "import {% endraw %}{{ package_name }}{% raw %}" create-tag: @@ -195,4 +197,52 @@ jobs: - name: Confirm tag not already present run: python3 ./.github/workflows/git_tag.py --confirm-tag-not-present - name: Create tag - run: python3 ./.github/workflows/git_tag.py --push-tag-to-remote{% endraw %} + run: python3 ./.github/workflows/git_tag.py --push-tag-to-remote + + publish-to-primary: + name: Publish Python distribution to Primary Package Registry + needs: [ build ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + environment: + name: pypi + url: https://pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} + permissions: + attestations: write + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@{% endraw %}{{ gha_download_artifact }}{% raw %} + with: + name: python-package-distributions + path: dist/ +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Publishing to CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + + - name: Publish distribution to Code Artifact + run: | + . .devcontainer/code-artifact-auth.sh + uv publish --verbose --index code-artifact-primary --username aws --password "$TWINE_PASSWORD" + +{% endraw %}{% else %}{% raw %} + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@v1.12.4 + with: + attestations: false{% endraw %}{% endif %}{% raw %} + + install-from-primary: + name: Install package from primary registry + needs: [ publish-to-primary, get-values ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + - name: Install from primary registry + run: pip install {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package_version }} + - name: Confirm library can be imported successfully + run: python -c "import {% endraw %}{{ package_name }}{% raw %}"{% endraw %} From 48ebebb339ec6f399f3b1fd729d527b2d039d168 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:49:16 +0000 Subject: [PATCH 10/20] actualy echo --- template/.github/workflows/publish.yaml.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 7268829b..1e51a07f 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -25,6 +25,7 @@ jobs: - name: Extract package version run: | VERSION=$(python3 ./.github/workflows/git_tag.py) + echo "Extracted version: $VERSION" echo "package_version=$VERSION" >> $GITHUB_OUTPUT lint: From bd5a32002201b9591496b11c02aaf2f25d8a0add Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:52:31 +0000 Subject: [PATCH 11/20] ordering --- template/.github/workflows/publish.yaml.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 1e51a07f..48a3fcb1 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -202,7 +202,7 @@ jobs: publish-to-primary: name: Publish Python distribution to Primary Package Registry - needs: [ build ] + needs: [ create-tag ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} environment: name: pypi From f84b305f70624ea1d30f07fed4e46f8f0a1c27d6 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:55:18 +0000 Subject: [PATCH 12/20] sleep --- template/.github/workflows/publish.yaml.jinja | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 48a3fcb1..f4dfd3a6 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -179,6 +179,11 @@ jobs: uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} with: python-version: {% endraw %}{{ python_version }}{% raw %} +{% endraw %}{% if python_package_registry == "PyPI" %}{% raw %} + - name: Sleep to allow PyPI Index to update before proceeding to the next step + uses: juliangruber/sleep-action@v2.0.3 + with: + time: 60s{% endraw %}{% endif %}{% raw %} - name: Install from staging registry run: pip install -i https://test.pypi.org/simple/ {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package_version }} - name: Confirm library can be imported successfully @@ -243,6 +248,11 @@ jobs: uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} with: python-version: {% endraw %}{{ python_version }}{% raw %} +{% endraw %}{% if python_package_registry == "PyPI" %}{% raw %} + - name: Sleep to allow PyPI Index to update before proceeding to the next step + uses: juliangruber/sleep-action@v2.0.3 + with: + time: 60s{% endraw %}{% endif %}{% raw %} - name: Install from primary registry run: pip install {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package_version }} - name: Confirm library can be imported successfully From dd36c64951aa1110c57ca09e54f079c4a321ff32 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 13:56:16 +0000 Subject: [PATCH 13/20] debug --- template/.github/workflows/publish.yaml.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index f4dfd3a6..836d9884 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -64,7 +64,7 @@ jobs: run: pre-commit run -a test: - needs: [ lint ] + needs: [ lint, get-values ] strategy: matrix: os: @@ -82,6 +82,7 @@ jobs: runs-on: ${{ matrix.os }} steps: + - run: echo '${{ toJSON(needs.get-values.outputs) }}' - name: Checkout code uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} From 7dcb2dbbb5a78d07dd3f2159454a647be6401630 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 14:03:07 +0000 Subject: [PATCH 14/20] get values --- template/.github/workflows/publish.yaml.jinja | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 836d9884..49bd174b 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -1,8 +1,13 @@ -{% raw %}name: Publish to Production Package Registry +{% raw %}name: Publish to Package Registry on: workflow_dispatch: - + inputs: + publish_to_primary: + description: 'Publish to Primary Registry' + type: boolean + required: false + default: false env: PYTHONUNBUFFERED: True PRE_COMMIT_HOME: ${{ github.workspace }}/.precommit_cache @@ -15,6 +20,8 @@ jobs: get-values: name: Get Values runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + outputs: + package-version: ${{ steps.extract-package-version.outputs.package_version }} steps: - name: Checkout code uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} @@ -23,6 +30,7 @@ jobs: with: python-version: {% endraw %}{{ python_version }}{% raw %} - name: Extract package version + id: extract-package-version run: | VERSION=$(python3 ./.github/workflows/git_tag.py) echo "Extracted version: $VERSION" @@ -186,12 +194,13 @@ jobs: with: time: 60s{% endraw %}{% endif %}{% raw %} - name: Install from staging registry - run: pip install -i https://test.pypi.org/simple/ {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package_version }} + run: pip install -i https://test.pypi.org/simple/ {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package-version }} - name: Confirm library can be imported successfully run: python -c "import {% endraw %}{{ package_name }}{% raw %}" create-tag: name: Create the git tag + if: ${{ github.event.inputs.publish_to_primary }} needs: [ install-from-staging ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} steps: @@ -208,6 +217,7 @@ jobs: publish-to-primary: name: Publish Python distribution to Primary Package Registry + if: ${{ github.event.inputs.publish_to_primary }} needs: [ create-tag ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} environment: @@ -242,6 +252,7 @@ jobs: install-from-primary: name: Install package from primary registry + if: ${{ github.event.inputs.publish_to_primary }} needs: [ publish-to-primary, get-values ] runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} steps: @@ -255,6 +266,6 @@ jobs: with: time: 60s{% endraw %}{% endif %}{% raw %} - name: Install from primary registry - run: pip install {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package_version }} + run: pip install {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package-version }} - name: Confirm library can be imported successfully run: python -c "import {% endraw %}{{ package_name }}{% raw %}"{% endraw %} From b8c685e7a12ae66031e0dade971a7edf45899a26 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 14:22:31 +0000 Subject: [PATCH 15/20] remove debug --- template/.github/workflows/publish.yaml.jinja | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 49bd174b..bea8aca8 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -72,7 +72,7 @@ jobs: run: pre-commit run -a test: - needs: [ lint, get-values ] + needs: [ lint ] strategy: matrix: os: @@ -90,7 +90,6 @@ jobs: runs-on: ${{ matrix.os }} steps: - - run: echo '${{ toJSON(needs.get-values.outputs) }}' - name: Checkout code uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} From a82888e801cf15afca9787ff38ce28d0bdc20977 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 14:25:05 +0000 Subject: [PATCH 16/20] wip --- .copier-answers.yml | 2 +- .devcontainer/devcontainer.json | 2 +- .github/workflows/hash_git_files.py | 19 ++++++++++++++----- _typos.toml | 6 ++++++ extensions/context.py | 2 ++ template/.github/workflows/hash_git_files.py | 19 ++++++++++++++----- template/.github/workflows/publish.yaml.jinja | 8 ++++---- template/_typos.toml | 6 ++++++ 8 files changed, 48 insertions(+), 16 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 0b53934b..cbbf51eb 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.31 +_commit: v0.0.31-2-g117b721 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ce46b6c3..da4ca85f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -59,5 +59,5 @@ "initializeCommand": "sh .devcontainer/initialize-command.sh", "onCreateCommand": "sh .devcontainer/on-create-command.sh", "postStartCommand": "sh .devcontainer/post-start-command.sh" - // Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): bbebc7c9 + // Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): 2308a2ae # spellchecker:disable-line } diff --git a/.github/workflows/hash_git_files.py b/.github/workflows/hash_git_files.py index bead4e4f..f8bcf65e 100644 --- a/.github/workflows/hash_git_files.py +++ b/.github/workflows/hash_git_files.py @@ -10,6 +10,10 @@ " // Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): " ) +DEVCONTAINER_COMMENT_LINE_SUFFIX = ( + " # spellchecker:disable-line" # the typos hook can sometimes mess with the hash without this +) + def get_tracked_files(repo_path: Path) -> list[str]: """Return a list of files tracked by Git in the given repository folder, using the 'git ls-files' command.""" @@ -76,9 +80,13 @@ def find_devcontainer_hash_line(lines: list[str]) -> tuple[int, str | None]: for i in range(len(lines) - 1, -1, -1): if lines[i].strip() == "}": # Check the line above it - if i > 0 and lines[i - 1].startswith(DEVCONTAINER_COMMENT_LINE_PREFIX): - current_hash = lines[i - 1].split(": ", 1)[1].strip() - return i - 1, current_hash + if i > 0: + above_line = lines[i - 1] + if above_line.startswith(DEVCONTAINER_COMMENT_LINE_PREFIX): + part_after_prefix = above_line.split(": ", 1)[1] + part_before_suffix = part_after_prefix.split("#")[0] + current_hash = part_before_suffix.strip() + return i - 1, current_hash return i, None return -1, None @@ -102,12 +110,13 @@ def update_devcontainer_context_hash(devcontainer_json_file: Path, new_hash: str lines = file.readlines() line_index, current_hash = find_devcontainer_hash_line(lines) + new_hash_line = f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}{DEVCONTAINER_COMMENT_LINE_SUFFIX}\n" if current_hash is not None: # Replace the old hash with the new hash - lines[line_index] = f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}\n" + lines[line_index] = new_hash_line else: # Insert the new hash line above the closing `}` - lines.insert(line_index, f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}\n") + lines.insert(line_index, new_hash_line) # Write the updated lines back to the file with devcontainer_json_file.open("w", encoding="utf-8") as file: diff --git a/_typos.toml b/_typos.toml index 8f351161..67a1a218 100644 --- a/_typos.toml +++ b/_typos.toml @@ -1,3 +1,9 @@ +[default] +extend-ignore-re = [ +# Line ignore with trailing: # spellchecker:disable-line +"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$" +] + [default.extend-words] # Words managed by the base template # `astroid` is the name of a python library, and it is used in pylint configuration. should not be corrected to asteroid diff --git a/extensions/context.py b/extensions/context.py index 734ac34e..8237088d 100644 --- a/extensions/context.py +++ b/extensions/context.py @@ -52,6 +52,8 @@ def hook(self, context: dict[Any, Any]) -> dict[Any, Any]: context["gha_setup_node"] = "v4.3.0" context["gha_action_gh_release"] = "v2.2.1" context["gha_mutex"] = "1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10" + context["gha_pypi_publish"] = v1.12.4" + context["gha_sleep"] = "v2.0.3" context["gha_linux_runner"] = "ubuntu-24.04" context["gha_windows_runner"] = "windows-2025" diff --git a/template/.github/workflows/hash_git_files.py b/template/.github/workflows/hash_git_files.py index bead4e4f..f8bcf65e 100644 --- a/template/.github/workflows/hash_git_files.py +++ b/template/.github/workflows/hash_git_files.py @@ -10,6 +10,10 @@ " // Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): " ) +DEVCONTAINER_COMMENT_LINE_SUFFIX = ( + " # spellchecker:disable-line" # the typos hook can sometimes mess with the hash without this +) + def get_tracked_files(repo_path: Path) -> list[str]: """Return a list of files tracked by Git in the given repository folder, using the 'git ls-files' command.""" @@ -76,9 +80,13 @@ def find_devcontainer_hash_line(lines: list[str]) -> tuple[int, str | None]: for i in range(len(lines) - 1, -1, -1): if lines[i].strip() == "}": # Check the line above it - if i > 0 and lines[i - 1].startswith(DEVCONTAINER_COMMENT_LINE_PREFIX): - current_hash = lines[i - 1].split(": ", 1)[1].strip() - return i - 1, current_hash + if i > 0: + above_line = lines[i - 1] + if above_line.startswith(DEVCONTAINER_COMMENT_LINE_PREFIX): + part_after_prefix = above_line.split(": ", 1)[1] + part_before_suffix = part_after_prefix.split("#")[0] + current_hash = part_before_suffix.strip() + return i - 1, current_hash return i, None return -1, None @@ -102,12 +110,13 @@ def update_devcontainer_context_hash(devcontainer_json_file: Path, new_hash: str lines = file.readlines() line_index, current_hash = find_devcontainer_hash_line(lines) + new_hash_line = f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}{DEVCONTAINER_COMMENT_LINE_SUFFIX}\n" if current_hash is not None: # Replace the old hash with the new hash - lines[line_index] = f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}\n" + lines[line_index] = new_hash_line else: # Insert the new hash line above the closing `}` - lines.insert(line_index, f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}\n") + lines.insert(line_index, new_hash_line) # Write the updated lines back to the file with devcontainer_json_file.open("w", encoding="utf-8") as file: diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index bea8aca8..e4ab719f 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -171,7 +171,7 @@ jobs: {% endraw %}{% else %}{% raw %} - name: Publish distribution to Test PyPI - uses: pypa/gh-action-pypi-publish@v1.12.4 + uses: pypa/gh-action-pypi-publish@{% endraw %}{{ gha_pypi_publish }}{% raw %} with: attestations: false repository-url: https://test.pypi.org/legacy/ @@ -189,7 +189,7 @@ jobs: python-version: {% endraw %}{{ python_version }}{% raw %} {% endraw %}{% if python_package_registry == "PyPI" %}{% raw %} - name: Sleep to allow PyPI Index to update before proceeding to the next step - uses: juliangruber/sleep-action@v2.0.3 + uses: juliangruber/sleep-action@{% endraw %}{{ gha_sleep }}{% raw %} with: time: 60s{% endraw %}{% endif %}{% raw %} - name: Install from staging registry @@ -245,7 +245,7 @@ jobs: {% endraw %}{% else %}{% raw %} - name: Publish distribution to PyPI - uses: pypa/gh-action-pypi-publish@v1.12.4 + uses: pypa/gh-action-pypi-publish@{% endraw %}{{ gha_pypi_publish }}{% raw %} with: attestations: false{% endraw %}{% endif %}{% raw %} @@ -261,7 +261,7 @@ jobs: python-version: {% endraw %}{{ python_version }}{% raw %} {% endraw %}{% if python_package_registry == "PyPI" %}{% raw %} - name: Sleep to allow PyPI Index to update before proceeding to the next step - uses: juliangruber/sleep-action@v2.0.3 + uses: juliangruber/sleep-action@{% endraw %}{{ gha_sleep }}{% raw %} with: time: 60s{% endraw %}{% endif %}{% raw %} - name: Install from primary registry diff --git a/template/_typos.toml b/template/_typos.toml index 13e4f4f5..8de2390d 100644 --- a/template/_typos.toml +++ b/template/_typos.toml @@ -1,3 +1,9 @@ +[default] +extend-ignore-re = [ +# Line ignore with trailing: # spellchecker:disable-line +"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$" +] + [default.extend-words] # Words managed by the base template # `astroid` is the name of a python library, and it is used in pylint configuration. should not be corrected to asteroid From 7754948c85b752d2af19d9bd3c67fae07d2dca40 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 14:26:32 +0000 Subject: [PATCH 17/20] copier --- .copier-answers.yml | 2 +- extensions/context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index cbbf51eb..19e64566 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.31-2-g117b721 +_commit: v0.0.31-3-gb915a33 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/extensions/context.py b/extensions/context.py index 8237088d..c5e91a9e 100644 --- a/extensions/context.py +++ b/extensions/context.py @@ -52,7 +52,7 @@ def hook(self, context: dict[Any, Any]) -> dict[Any, Any]: context["gha_setup_node"] = "v4.3.0" context["gha_action_gh_release"] = "v2.2.1" context["gha_mutex"] = "1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10" - context["gha_pypi_publish"] = v1.12.4" + context["gha_pypi_publish"] = "v1.12.4" context["gha_sleep"] = "v2.0.3" context["gha_linux_runner"] = "ubuntu-24.04" context["gha_windows_runner"] = "windows-2025" From 9a98010204db4d877cd3df9ac285c7de6e778bef Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 14:34:20 +0000 Subject: [PATCH 18/20] snake --- template/.github/workflows/publish.yaml.jinja | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index e4ab719f..07313bdd 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -195,7 +195,7 @@ jobs: - name: Install from staging registry run: pip install -i https://test.pypi.org/simple/ {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package-version }} - name: Confirm library can be imported successfully - run: python -c "import {% endraw %}{{ package_name }}{% raw %}" + run: python -c "import {% endraw %}{{ package_name | replace('-', '_') }}{% raw %}" create-tag: name: Create the git tag @@ -267,4 +267,4 @@ jobs: - name: Install from primary registry run: pip install {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package-version }} - name: Confirm library can be imported successfully - run: python -c "import {% endraw %}{{ package_name }}{% raw %}"{% endraw %} + run: python -c "import {% endraw %}{{ package_name | replace('-', '_') }}{% raw %}"{% endraw %} From d0ae779d2251aa39a016a2d438b268a2dbce4f94 Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 14:34:45 +0000 Subject: [PATCH 19/20] tag --- .copier-answers.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 19e64566..0dfaa61b 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.31-3-gb915a33 +_commit: v0.0.32 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: From 17c0a8a8021c6a117f3871297fb60c315c67990f Mon Sep 17 00:00:00 2001 From: Eli Fine Date: Fri, 2 May 2025 14:35:47 +0000 Subject: [PATCH 20/20] readme --- template/README.md.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/README.md.jinja b/template/README.md.jinja index fe0531be..9d256714 100644 --- a/template/README.md.jinja +++ b/template/README.md.jinja @@ -16,7 +16,7 @@ Documentation is hosted on [ReadTheDocs](https://{% endraw %}{{ package_name }}{ # Development This project has a dev container. If you already have VS Code and Docker installed, you can click the badge above or [here](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url={% endraw %}{{ full_repo_url }}{% raw %}) to get started. Clicking these links will cause VS Code to automatically install the Dev Containers extension if needed, clone the source code into a container volume, and spin up a dev container for use. - +To publish a new version of the repository, you can run the `Publish` workflow manually and publish to the staging registry from any branch, and you can check the 'Publish to Primary' option when on `main` to publish to the primary registry and create a git tag.