Skip to content

Commit c279c9d

Browse files
authored
Install from code artifact (#12)
## Why is this change necessary? Need a configuration option to be able to install from CodeArtifact ## How does this change address the issue? Adds a step in CI to authenticate with the OIDC role. Creates scripts to be able to authenticate locally. Updates pyproject.toml to specify CodeArtifact ## What side effects does this change have? None ## How is this change tested? In downstream repos that use CodeArtifact and those that don't For CI in this repo, it was deemed prohibitive to try and actively install from CodeArtifact, so there's a script that updates any instantiated templates pyproject.toml to point it back to PyPI
1 parent fa4aa24 commit c279c9d

21 files changed

+327
-36
lines changed

.copier-answers.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Changes here will be overwritten by Copier
2-
_commit: v0.0.6-6-g2b24a38
2+
_commit: v0.0.7-44-gea357db
33
_src_path: gh:LabAutomationAndScreening/copier-base-template.git
44
description: Copier template for creating Python libraries and executables
55
python_ci_versions:

.devcontainer/create-aws-profile.sh

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/sh
1+
#!/usr/bin/env sh
22
set -ex
33

44
mkdir -p ~/.aws
@@ -10,10 +10,16 @@ else
1010
fi
1111

1212
cat >> ~/.aws/config <<EOF
13+
14+
15+
16+
1317
[profile localstack]
1418
region=us-east-1
1519
output=json
1620
endpoint_url = $LOCALSTACK_ENDPOINT_URL
21+
22+
1723
EOF
1824
cat >> ~/.aws/credentials <<EOF
1925
[localstack]

.devcontainer/manual-setup-deps.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,48 @@
11
#!/usr/bin/env sh
22
# can pass in the full major.minor.patch version of python as an optional argument
33
# can set `--skip-lock` as optional argument to just install dependencies without verifying lock file
4+
# can set `--optionally-lock` to check for a uv.lock file in the project directory and only respect the lock if it already exists (useful for initially instantiating the repository) (mutually exclusive with --skip-lock)
5+
46
set -ex
57

68
# Ensure that uv won't use the default system Python
79
python_version="3.12.7"
810

911
# Parse arguments
1012
skip_lock=false
13+
optionally_lock=false
1114
while [ "$#" -gt 0 ]; do
1215
case $1 in
1316
--skip-lock) skip_lock=true ;;
17+
--optionally-lock) optionally_lock=true ;;
1418
*) python_version="${1:-$python_version}" ;; # Take the first non-flag argument as the input
1519
esac
1620
shift
1721
done
1822

23+
# Ensure that --skip-lock and --optionally-lock are mutually exclusive
24+
if [ "$skip_lock" = "true" ] && [ "$optionally_lock" = "true" ]; then
25+
echo "Error: --skip-lock and --optionally-lock cannot be used together." >&2
26+
exit 1
27+
fi
28+
1929
export UV_PYTHON="$python_version"
2030
export UV_PYTHON_PREFERENCE=only-system
2131

2232
SCRIPT_DIR="$(dirname "$0")"
2333
PROJECT_ROOT_DIR="$(realpath "$SCRIPT_DIR/..")"
2434

35+
# If optionally_lock is set, decide whether to skip locking based on the presence of uv.lock
36+
if [ "$optionally_lock" = "true" ]; then
37+
if [ ! -f "$PROJECT_ROOT_DIR/uv.lock" ]; then
38+
skip_lock=true
39+
else
40+
skip_lock=false
41+
fi
42+
fi
43+
44+
45+
2546
# Ensure that the lock file is in a good state
2647
if [ "$skip_lock" = "false" ]; then
2748
uv lock --check --directory "$PROJECT_ROOT_DIR"

.devcontainer/on-create-command.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@ git config --global --add safe.directory /workspaces/copier-python-package-templ
77

88
sh .devcontainer/on-create-command-boilerplate.sh
99

10-
sh .devcontainer/manual-setup-deps.sh
11-
1210
pre-commit install --install-hooks
11+
12+
sh .devcontainer/manual-setup-deps.sh --optionally-lock

.github/actions/install_deps_uv/action.yml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,19 @@ inputs:
1414
description: What's the relative path to the project?
1515
required: false
1616
default: ./
17+
code-artifact-auth-role-name:
18+
type: string
19+
description: What's the role name to use for CodeArtifact authentication?
20+
required: false
21+
default: no-code-artifact
22+
code-artifact-auth-role-account-id:
23+
type: string
24+
description: What's the AWS Account ID that the role is in?
25+
required: false
26+
code-artifact-auth-region:
27+
type: string
28+
description: What region should the role use?
29+
required: false
1730

1831

1932
runs:
@@ -41,12 +54,22 @@ runs:
4154
run: .github/actions/install_deps_uv/install-ci-tooling.ps1 ${{ env.PYTHON_VERSION }}
4255
shell: pwsh
4356

57+
- name: OIDC Auth for CodeArtifact
58+
if: ${{ inputs.code-artifact-auth-role-name != 'no-code-artifact' }}
59+
uses: aws-actions/configure-aws-credentials@v4.0.2
60+
with:
61+
role-to-assume: arn:aws:iam::${{ inputs.code-artifact-auth-role-account-id }}:role/${{ inputs.code-artifact-auth-role-name }}
62+
aws-region: ${{ inputs.code-artifact-auth-region }}
63+
4464
- name: Install Dependencies (Linux)
4565
if: ${{ inputs.uv-sync && runner.os == 'Linux' }}
4666
run: |
4767
sh .devcontainer/manual-setup-deps.sh ${{ env.PYTHON_VERSION }}
4868
shell: bash
4969

70+
71+
72+
5073
- name: Install Dependencies (Windows)
5174
if: ${{ inputs.uv-sync && runner.os == 'Windows' }}
5275
run: .github/actions/install_deps_uv/manual-setup-deps.ps1 ${{ env.PYTHON_VERSION }}

.github/workflows/ci.yaml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ jobs:
7777
- name: Checkout code
7878
uses: actions/checkout@v4.2.2
7979

80+
- name: Move python script that replaces private package registry information to temp folder so it doesn't get deleted
81+
run: |
82+
mv .github/workflows/replace_private_package_registries.py $RUNNER_TEMP
83+
8084
- name: Install python tooling
8185
uses: ./.github/actions/install_deps_uv
8286
with:
@@ -107,7 +111,12 @@ jobs:
107111
108112
109113
- name: install new dependencies
114+
env:
115+
CODEARTIFACT_AUTH_TOKEN: 'faketoken'
116+
UV_NO_CACHE: 'true'
110117
run: |
118+
# Remove any specification of a Python repository having a default other than PyPI...because in this CI pipeline we can only install from PyPI
119+
python $RUNNER_TEMP/replace_private_package_registries.py
111120
sh .devcontainer/manual-setup-deps.sh ${{ matrix.python-version }} --skip-lock
112121
# Add everything to git so that pre-commit recognizes the files and runs on them
113122
git add .
Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
"""Update any project files that point to a private package registry to use public ones.
2+
3+
Since the CI pipelines for testing these copier templates don't have access to private registries, we can't test installing from them as part of CI.
4+
5+
Seems minimal risk, since the only problem we'd be missing is if the pyproject.toml (or similar config files) had syntax errors that would have been
6+
caught by pre-commit.
7+
"""
8+
9+
import re
10+
from pathlib import Path
11+
12+
13+
def process_file(file_path: Path):
14+
# Read the entire file content
15+
content = file_path.read_text()
16+
17+
# Regex to match a block starting with [[tool.uv.index]]
18+
# until the next block header (a line starting with [[) or the end of the file.
19+
pattern = re.compile(r"(\[\[tool\.uv\.index\]\].*?)(?=\n\[\[|$)", re.DOTALL)
20+
21+
# Find all uv.index blocks.
22+
blocks = pattern.findall(content)
23+
24+
# Check if any block contains "default = true"
25+
if not any("default = true" in block for block in blocks):
26+
print(f"No changes in: {file_path}")
27+
return
28+
29+
# If at least one block contains "default = true", remove all uv.index blocks.
30+
new_content = pattern.sub("", content)
31+
32+
# Ensure file ends with a newline before appending the new block.
33+
if not new_content.endswith("\n"):
34+
new_content += "\n"
35+
36+
# Append the new block.
37+
new_block = '[[tool.uv.index]]\nname = "pypi"\nurl = "https://pypi.org/simple/"\n'
38+
new_content += new_block
39+
40+
# Write the updated content back to the file.
41+
_ = file_path.write_text(new_content)
42+
print(f"Updated file: {file_path}")
43+
44+
45+
def main():
46+
base_dir = Path(".")
47+
# Use rglob to find all pyproject.toml files recursively.
48+
for file_path in base_dir.rglob("pyproject.toml"):
49+
# Check if the file is at most two levels deep.
50+
# The relative path's parts count should be <= 3 (e.g. "pyproject.toml" is 1 part,
51+
# "subdir/pyproject.toml" is 2 parts, and "subdir/subsubdir/pyproject.toml" is 3 parts).
52+
if len(file_path.relative_to(base_dir).parts) <= 3:
53+
process_file(file_path)
54+
55+
56+
if __name__ == "__main__":
57+
main()

copier.yml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,14 @@ python_version:
2626
help: What version of Python is used for development?
2727
default: "3.12.7"
2828

29+
python_package_registry:
30+
type: str
31+
help: What registry should Python Packgaes be installed from?
32+
choices:
33+
- PyPI
34+
- AWS CodeArtifact
35+
default: PyPI
36+
2937
python_ci_versions:
3038
type: str
3139
help: What versions should Python run CI on the instantiated template?
@@ -35,6 +43,31 @@ python_ci_versions:
3543
- "3.13.2"
3644

3745

46+
aws_identity_center_id:
47+
type: str
48+
help: What's the ID of your Organization's AWS Identity center, e.g. d-9145c20053?
49+
when: "{{ python_package_registry == 'AWS CodeArtifact' }}"
50+
51+
aws_org_home_region:
52+
type: str
53+
help: What is the home region of the AWS Organization (where all of the central infrastructure is deployed)?
54+
default: us-east-1
55+
when: "{{ python_package_registry == 'AWS CodeArtifact' }}"
56+
57+
aws_central_infrastructure_account_id:
58+
type: str
59+
help: What's the ID of your Organization's AWS Account containing Central Infrastructure (e.g. CodeArtifact)?
60+
when: "{{ python_package_registry == 'AWS CodeArtifact' }}"
61+
62+
core_infra_base_access_profile_name:
63+
type: str
64+
help: What's the AWS Identity Center Profile name for base access to the Central Infrastructure account (i.e. to read from CodeArtifact)?
65+
when: "{{ python_package_registry == 'AWS CodeArtifact' }}"
66+
default: CoreInfraBaseAccess
67+
68+
69+
70+
3871

3972
# Questions specific to this template
4073

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
{% if python_package_registry is defined and python_package_registry == "AWS CodeArtifact" %}{% raw %}#!/usr/bin/env bash
2+
set -ex
3+
4+
# If none of these are set we can't possibly continue and should fail so you can fix it
5+
if [ -z "$AWS_PROFILE" ] && [ -z "$AWS_ACCESS_KEY_ID" ] && [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then
6+
echo "No AWS profile, access key, or auth token found, cannot proceed."
7+
exit 1
8+
else
9+
# Only regenerate the token if it doesn't exist or wasn't already set as an environmental variable (e.g. during CI or passed into a docker image build)
10+
if [ -z "$CODEARTIFACT_AUTH_TOKEN" ]; then
11+
echo "Fetching CodeArtifact token"
12+
if [ -z "$CI" ]; then
13+
PROFILE_ARGS="--profile={% endraw %}{{ core_infra_base_access_profile_name }}{% raw %}"
14+
else
15+
PROFILE_ARGS=""
16+
fi
17+
18+
# Check if AWS credentials are valid by trying to retrieve the caller identity.
19+
# If the ARN is not returned, assume credentials are expired or not set correctly.
20+
caller_identity=$(aws sts get-caller-identity --region={% endraw %}{{ aws_org_home_region }}{% raw %} $PROFILE_ARGS --query Arn --output text 2>/dev/null || echo "")
21+
if [ -z "$caller_identity" ]; then
22+
if [ -n "$CI" ]; then
23+
echo "Error: In CI environment, aws sso login should never be called...something is wrong with this script or your workflow...perhaps you did not OIDC Auth yet in CI?"
24+
exit 1
25+
fi
26+
echo "SSO credentials not found or expired, logging in..."
27+
aws sso login $PROFILE_ARGS
28+
else
29+
echo "Using existing AWS credentials: $caller_identity"
30+
fi
31+
32+
set +x
33+
export CODEARTIFACT_AUTH_TOKEN=$(aws codeartifact get-authorization-token \
34+
--domain {% endraw %}{{ repo_org_name }}{% raw %} \
35+
--domain-owner {% endraw %}{{ aws_central_infrastructure_account_id }}{% raw %} \
36+
--region {% endraw %}{{ aws_org_home_region }}{% raw %} \
37+
--query authorizationToken \
38+
--output text $PROFILE_ARGS)
39+
set -x
40+
fi
41+
42+
set +x
43+
export UV_INDEX_CODE_ARTIFACT_PRIMARY_PASSWORD="$CODEARTIFACT_AUTH_TOKEN"
44+
export UV_INDEX_CODE_ARTIFACT_STAGING_PASSWORD="$CODEARTIFACT_AUTH_TOKEN"
45+
set -x
46+
47+
fi{% endraw %}{% else %}{% raw %}# Placeholder file not being used by these copier template answers{% endraw %}{% endif %}

template/.devcontainer/create-aws-profile.sh

Lines changed: 0 additions & 22 deletions
This file was deleted.

0 commit comments

Comments
 (0)