diff --git a/.copier-answers.yml b/.copier-answers.yml index 0b53934b..0dfaa61b 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v0.0.31 +_commit: v0.0.32 _src_path: gh:LabAutomationAndScreening/copier-base-template.git description: Copier template for creating Python libraries and executables python_ci_versions: diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ce46b6c3..da4ca85f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -59,5 +59,5 @@ "initializeCommand": "sh .devcontainer/initialize-command.sh", "onCreateCommand": "sh .devcontainer/on-create-command.sh", "postStartCommand": "sh .devcontainer/post-start-command.sh" - // Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): bbebc7c9 + // Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): 2308a2ae # spellchecker:disable-line } diff --git a/.github/workflows/hash_git_files.py b/.github/workflows/hash_git_files.py index bead4e4f..f8bcf65e 100644 --- a/.github/workflows/hash_git_files.py +++ b/.github/workflows/hash_git_files.py @@ -10,6 +10,10 @@ " // Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): " ) +DEVCONTAINER_COMMENT_LINE_SUFFIX = ( + " # spellchecker:disable-line" # the typos hook can sometimes mess with the hash without this +) + def get_tracked_files(repo_path: Path) -> list[str]: """Return a list of files tracked by Git in the given repository folder, using the 'git ls-files' command.""" @@ -76,9 +80,13 @@ def find_devcontainer_hash_line(lines: list[str]) -> tuple[int, str | None]: for i in range(len(lines) - 1, -1, -1): if lines[i].strip() == "}": # Check the line above it - if i > 0 and lines[i - 1].startswith(DEVCONTAINER_COMMENT_LINE_PREFIX): - current_hash = lines[i - 1].split(": ", 1)[1].strip() - return i - 1, current_hash + if i > 0: + above_line = lines[i - 1] + if above_line.startswith(DEVCONTAINER_COMMENT_LINE_PREFIX): + part_after_prefix = above_line.split(": ", 1)[1] + part_before_suffix = part_after_prefix.split("#")[0] + current_hash = part_before_suffix.strip() + return i - 1, current_hash return i, None return -1, None @@ -102,12 +110,13 @@ def update_devcontainer_context_hash(devcontainer_json_file: Path, new_hash: str lines = file.readlines() line_index, current_hash = find_devcontainer_hash_line(lines) + new_hash_line = f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}{DEVCONTAINER_COMMENT_LINE_SUFFIX}\n" if current_hash is not None: # Replace the old hash with the new hash - lines[line_index] = f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}\n" + lines[line_index] = new_hash_line else: # Insert the new hash line above the closing `}` - lines.insert(line_index, f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}\n") + lines.insert(line_index, new_hash_line) # Write the updated lines back to the file with devcontainer_json_file.open("w", encoding="utf-8") as file: diff --git a/_typos.toml b/_typos.toml index 8f351161..67a1a218 100644 --- a/_typos.toml +++ b/_typos.toml @@ -1,3 +1,9 @@ +[default] +extend-ignore-re = [ +# Line ignore with trailing: # spellchecker:disable-line +"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$" +] + [default.extend-words] # Words managed by the base template # `astroid` is the name of a python library, and it is used in pylint configuration. should not be corrected to asteroid diff --git a/extensions/context.py b/extensions/context.py index 734ac34e..c5e91a9e 100644 --- a/extensions/context.py +++ b/extensions/context.py @@ -52,6 +52,8 @@ def hook(self, context: dict[Any, Any]) -> dict[Any, Any]: context["gha_setup_node"] = "v4.3.0" context["gha_action_gh_release"] = "v2.2.1" context["gha_mutex"] = "1ebad517141198e08d47cf72f3c0975316620a65 # v1.0.0-alpha.10" + context["gha_pypi_publish"] = "v1.12.4" + context["gha_sleep"] = "v2.0.3" context["gha_linux_runner"] = "ubuntu-24.04" context["gha_windows_runner"] = "windows-2025" diff --git a/src/git_tag.py b/src/git_tag.py new file mode 100644 index 00000000..e0bd5abf --- /dev/null +++ b/src/git_tag.py @@ -0,0 +1,95 @@ +import argparse +import subprocess +import tomllib +from pathlib import Path + + +def extract_version(toml_path: Path | str) -> str: + """Load toml_path and return the version string. + + Checks [project].version (PEP 621) first, then [tool.poetry].version. Raises KeyError if no version field is found. + """ + path = Path(toml_path) + with path.open("rb") as f: + data = tomllib.load(f) + + project = data.get("project", {}) + if version := project.get("version"): + return version + + tool = data.get("tool", {}) + poetry = tool.get("poetry", {}) + if version := poetry.get("version"): + return version + + raise KeyError(f"No version field found in {path!r}") # noqa: TRY003 # not worth a custom exception + + +def ensure_tag_not_present(tag: str, remote: str) -> None: + try: + _ = subprocess.run( # noqa: S603 # this is trusted input, it's our own arguments being passed in + ["git", "ls-remote", "--exit-code", "--tags", remote, f"refs/tags/{tag}"], # noqa: S607 # if `git` isn't in PATH already, then there are bigger problems to solve + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + raise Exception(f"Error: tag '{tag}' exists on remote '{remote}'") # noqa: TRY002,TRY003 # not worth a custom exception + except subprocess.CalledProcessError: + # tag not present, continue + return + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Extract the version from a pyproject.toml file, " + "confirm that git tag v is not present, or " + "create and push the tag to a remote." + ) + ) + _ = parser.add_argument( + "file", + nargs="?", + default="pyproject.toml", + help="Path to pyproject.toml (default: pyproject.toml)", + ) + _ = parser.add_argument( + "--confirm-tag-not-present", + action="store_true", + help=("Check that git tag v is NOT present on the remote. If the tag exists, exit with an error."), + ) + _ = parser.add_argument( + "--push-tag-to-remote", + action="store_true", + help=( + "Create git tag v locally and push it to the remote. " + "Internally confirms the tag is not already present." + ), + ) + _ = parser.add_argument( + "--remote", + default="origin", + help="Name of git remote to query/push (default: origin)", + ) + args = parser.parse_args() + + ver = extract_version(args.file) + + tag = f"v{ver}" + + if args.push_tag_to_remote: + ensure_tag_not_present(tag, args.remote) + _ = subprocess.run(["git", "tag", tag], check=True) # noqa: S603,S607 # this is trusted input, it's our own pyproject.toml file. and if `git` isn't in PATH, then there are larger problems anyway + _ = subprocess.run(["git", "push", args.remote, tag], check=True) # noqa: S603,S607 # this is trusted input, it's our own pyproject.toml file. and if `git` isn't in PATH, then there are larger problems anyway + return + + if args.confirm_tag_not_present: + ensure_tag_not_present(tag, args.remote) + return + + # Default behavior: just print the version + print(ver) # noqa: T201 # specifically printing this out so CI pipelines can read the value from stdout + + +if __name__ == "__main__": + main() diff --git a/template/.github/workflows/git_tag.py b/template/.github/workflows/git_tag.py new file mode 120000 index 00000000..3f6f11c3 --- /dev/null +++ b/template/.github/workflows/git_tag.py @@ -0,0 +1 @@ +../../../src/git_tag.py \ No newline at end of file diff --git a/template/.github/workflows/hash_git_files.py b/template/.github/workflows/hash_git_files.py index bead4e4f..f8bcf65e 100644 --- a/template/.github/workflows/hash_git_files.py +++ b/template/.github/workflows/hash_git_files.py @@ -10,6 +10,10 @@ " // Devcontainer context hash (do not manually edit this, it's managed by a pre-commit hook): " ) +DEVCONTAINER_COMMENT_LINE_SUFFIX = ( + " # spellchecker:disable-line" # the typos hook can sometimes mess with the hash without this +) + def get_tracked_files(repo_path: Path) -> list[str]: """Return a list of files tracked by Git in the given repository folder, using the 'git ls-files' command.""" @@ -76,9 +80,13 @@ def find_devcontainer_hash_line(lines: list[str]) -> tuple[int, str | None]: for i in range(len(lines) - 1, -1, -1): if lines[i].strip() == "}": # Check the line above it - if i > 0 and lines[i - 1].startswith(DEVCONTAINER_COMMENT_LINE_PREFIX): - current_hash = lines[i - 1].split(": ", 1)[1].strip() - return i - 1, current_hash + if i > 0: + above_line = lines[i - 1] + if above_line.startswith(DEVCONTAINER_COMMENT_LINE_PREFIX): + part_after_prefix = above_line.split(": ", 1)[1] + part_before_suffix = part_after_prefix.split("#")[0] + current_hash = part_before_suffix.strip() + return i - 1, current_hash return i, None return -1, None @@ -102,12 +110,13 @@ def update_devcontainer_context_hash(devcontainer_json_file: Path, new_hash: str lines = file.readlines() line_index, current_hash = find_devcontainer_hash_line(lines) + new_hash_line = f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}{DEVCONTAINER_COMMENT_LINE_SUFFIX}\n" if current_hash is not None: # Replace the old hash with the new hash - lines[line_index] = f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}\n" + lines[line_index] = new_hash_line else: # Insert the new hash line above the closing `}` - lines.insert(line_index, f"{DEVCONTAINER_COMMENT_LINE_PREFIX}{new_hash}\n") + lines.insert(line_index, new_hash_line) # Write the updated lines back to the file with devcontainer_json_file.open("w", encoding="utf-8") as file: diff --git a/template/.github/workflows/publish.yaml.jinja b/template/.github/workflows/publish.yaml.jinja index 39c0ddda..07313bdd 100644 --- a/template/.github/workflows/publish.yaml.jinja +++ b/template/.github/workflows/publish.yaml.jinja @@ -1,8 +1,13 @@ -{% raw %}name: Publish to Production Package Registry +{% raw %}name: Publish to Package Registry on: workflow_dispatch: - + inputs: + publish_to_primary: + description: 'Publish to Primary Registry' + type: boolean + required: false + default: false env: PYTHONUNBUFFERED: True PRE_COMMIT_HOME: ${{ github.workspace }}/.precommit_cache @@ -12,6 +17,25 @@ permissions: contents: write # needed for mutex jobs: + get-values: + name: Get Values + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + outputs: + package-version: ${{ steps.extract-package-version.outputs.package_version }} + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + - name: Extract package version + id: extract-package-version + run: | + VERSION=$(python3 ./.github/workflows/git_tag.py) + echo "Extracted version: $VERSION" + echo "package_version=$VERSION" >> $GITHUB_OUTPUT + lint: name: Pre-commit runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} @@ -109,6 +133,30 @@ jobs: {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} uv build --no-sources + - name: Upload build package + uses: actions/upload-artifact@{% endraw %}{{ gha_upload_artifact }}{% raw %} + with: + name: python-package-distributions + path: dist/ + if-no-files-found: error + + + publish-to-staging: + name: Publish Python distribution to Staging Package Registry + needs: [ build ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + environment: + name: testpypi + url: https://test.pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} + permissions: + attestations: write + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@{% endraw %}{{ gha_download_artifact }}{% raw %} + with: + name: python-package-distributions + path: dist/ {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} - name: OIDC Auth for Publishing to CodeArtifact uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} @@ -116,9 +164,107 @@ jobs: role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + - name: Publish distribution to Code Artifact + run: | + . .devcontainer/code-artifact-auth.sh + uv publish --verbose --index code-artifact-primary --username aws --password "$TWINE_PASSWORD" + +{% endraw %}{% else %}{% raw %} + - name: Publish distribution to Test PyPI + uses: pypa/gh-action-pypi-publish@{% endraw %}{{ gha_pypi_publish }}{% raw %} + with: + attestations: false + repository-url: https://test.pypi.org/legacy/ {% endraw %}{% endif %}{% raw %} - - name: Publish package + + install-from-staging: + name: Install package from staging registry + needs: [ publish-to-staging, get-values ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} +{% endraw %}{% if python_package_registry == "PyPI" %}{% raw %} + - name: Sleep to allow PyPI Index to update before proceeding to the next step + uses: juliangruber/sleep-action@{% endraw %}{{ gha_sleep }}{% raw %} + with: + time: 60s{% endraw %}{% endif %}{% raw %} + - name: Install from staging registry + run: pip install -i https://test.pypi.org/simple/ {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package-version }} + - name: Confirm library can be imported successfully + run: python -c "import {% endraw %}{{ package_name | replace('-', '_') }}{% raw %}" + + create-tag: + name: Create the git tag + if: ${{ github.event.inputs.publish_to_primary }} + needs: [ install-from-staging ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Checkout code + uses: actions/checkout@{% endraw %}{{ gha_checkout }}{% raw %} + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} + - name: Confirm tag not already present + run: python3 ./.github/workflows/git_tag.py --confirm-tag-not-present + - name: Create tag + run: python3 ./.github/workflows/git_tag.py --push-tag-to-remote + + publish-to-primary: + name: Publish Python distribution to Primary Package Registry + if: ${{ github.event.inputs.publish_to_primary }} + needs: [ create-tag ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + environment: + name: pypi + url: https://pypi.org/p/{% endraw %}{{ package_name | replace('_', '-') }}{% raw %} + permissions: + attestations: write + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@{% endraw %}{{ gha_download_artifact }}{% raw %} + with: + name: python-package-distributions + path: dist/ +{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} + - name: OIDC Auth for Publishing to CodeArtifact + uses: aws-actions/configure-aws-credentials@{% endraw %}{{ gha_configure_aws_credentials }}{% raw %} + with: + role-to-assume: arn:aws:iam::{% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %}:role/GHA-CA-Primary-{% endraw %}{{ repo_name }}{% raw %} + aws-region: {% endraw %}{{ aws_org_home_region }}{% raw %} + + - name: Publish distribution to Code Artifact run: | -{% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}{% raw %} . .devcontainer/code-artifact-auth.sh{% endraw %}{% endif %}{% raw %} - uv publish --verbose --index {% endraw %}{% if python_package_registry == "AWS CodeArtifact" %}code-artifact-primary --username aws --password "$TWINE_PASSWORD"{% else %}pypi{% endif %} + . .devcontainer/code-artifact-auth.sh + uv publish --verbose --index code-artifact-primary --username aws --password "$TWINE_PASSWORD" + +{% endraw %}{% else %}{% raw %} + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@{% endraw %}{{ gha_pypi_publish }}{% raw %} + with: + attestations: false{% endraw %}{% endif %}{% raw %} + + install-from-primary: + name: Install package from primary registry + if: ${{ github.event.inputs.publish_to_primary }} + needs: [ publish-to-primary, get-values ] + runs-on: {% endraw %}{{ gha_linux_runner }}{% raw %} + steps: + - name: Setup python + uses: actions/setup-python@{% endraw %}{{ gha_setup_python }}{% raw %} + with: + python-version: {% endraw %}{{ python_version }}{% raw %} +{% endraw %}{% if python_package_registry == "PyPI" %}{% raw %} + - name: Sleep to allow PyPI Index to update before proceeding to the next step + uses: juliangruber/sleep-action@{% endraw %}{{ gha_sleep }}{% raw %} + with: + time: 60s{% endraw %}{% endif %}{% raw %} + - name: Install from primary registry + run: pip install {% endraw %}{{ package_name | replace('_', '-') }}{% raw %}==${{ needs.get-values.outputs.package-version }} + - name: Confirm library can be imported successfully + run: python -c "import {% endraw %}{{ package_name | replace('-', '_') }}{% raw %}"{% endraw %} diff --git a/template/README.md.jinja b/template/README.md.jinja index fe0531be..9d256714 100644 --- a/template/README.md.jinja +++ b/template/README.md.jinja @@ -16,7 +16,7 @@ Documentation is hosted on [ReadTheDocs](https://{% endraw %}{{ package_name }}{ # Development This project has a dev container. If you already have VS Code and Docker installed, you can click the badge above or [here](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url={% endraw %}{{ full_repo_url }}{% raw %}) to get started. Clicking these links will cause VS Code to automatically install the Dev Containers extension if needed, clone the source code into a container volume, and spin up a dev container for use. - +To publish a new version of the repository, you can run the `Publish` workflow manually and publish to the staging registry from any branch, and you can check the 'Publish to Primary' option when on `main` to publish to the primary registry and create a git tag. diff --git a/template/_typos.toml b/template/_typos.toml index 13e4f4f5..8de2390d 100644 --- a/template/_typos.toml +++ b/template/_typos.toml @@ -1,3 +1,9 @@ +[default] +extend-ignore-re = [ +# Line ignore with trailing: # spellchecker:disable-line +"(?Rm)^.*(#|//)\\s*spellchecker:disable-line$" +] + [default.extend-words] # Words managed by the base template # `astroid` is the name of a python library, and it is used in pylint configuration. should not be corrected to asteroid