diff --git a/.github/workflows/build-root.yml b/.github/workflows/build-root.yml index 578a080..4d8e295 100644 --- a/.github/workflows/build-root.yml +++ b/.github/workflows/build-root.yml @@ -17,47 +17,62 @@ jobs: runs-on: ubuntu-latest outputs: pyenv_version: ${{ steps.get_pyenv_version.outputs.pyenv_version }} + resolved_versions: ${{ steps.resolve_versions.outputs.resolved_versions }} steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v7 + + - name: Checkout pyenv + uses: actions/checkout@v7 with: repository: pyenv/pyenv fetch-depth: 0 fetch-tags: true + path: pyenv - name: Get latest pyenv version id: get_pyenv_version run: | + cd pyenv echo "pyenv_version=$(git describe --abbrev=0 --tags)" >> $GITHUB_OUTPUT + - name: Resolve Python versions + id: resolve_versions + run: | + resolved_versions=$(PYENV_ROOT=./pyenv VERSIONS_TOML=./versions.toml python3 scripts/find_version.py) + # JSON array with objects containing "tag" and "version" keys + echo "resolved_versions=$resolved_versions" >> $GITHUB_OUTPUT + build-base: name: Build Python Builder Base needs: pyenv-version - outputs: - resolved_versions: ${{ steps.resolve_versions.outputs.resolved_versions }} runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v7 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to Github Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push Python builder base - uses: docker/build-push-action@v6 + - name: Build and push Python builder base (multi-platform) + uses: docker/build-push-action@v7 with: context: . - file: ./Dockerfile.base + file: ./Dockerfile + target: builder-base + platforms: linux/amd64,linux/arm64 push: ${{ github.ref == 'refs/heads/main' }} cache-to: type=inline - load: true cache-from: type=registry,ref=ghcr.io/python-discord/python-builds:builder-base build-args: | PYENV_VERSION=${{ needs.pyenv-version.outputs.pyenv_version }} @@ -65,20 +80,14 @@ jobs: ghcr.io/python-discord/python-builds:builder-base ghcr.io/python-discord/python-builds:builder-base-${{ needs.pyenv-version.outputs.pyenv_version }} - - name: Run Docker container to resolve versions - id: resolve_versions - run: | - resolved_versions=$(docker run --mount type=bind,src=./versions.toml,dst=/versions.toml --rm ghcr.io/python-discord/python-builds:builder-base python3 /scripts/find_version.py) - # JSON array with objects containing "tag" and "version" keys - echo "resolved_versions=$resolved_versions" >> $GITHUB_OUTPUT - build-versions: name: Build Python Versions - needs: build-base + needs: [pyenv-version, build-base] strategy: matrix: - version_info: ${{ fromJson(needs.build-base.outputs.resolved_versions) }} + version_info: ${{ fromJson(needs.pyenv-version.outputs.resolved_versions) }} uses: ./.github/workflows/build-version.yml with: version: ${{ matrix.version_info.version }} tag: ${{ matrix.version_info.tag }} + pyenv_version: ${{ needs.pyenv-version.outputs.pyenv_version }} diff --git a/.github/workflows/build-version.yml b/.github/workflows/build-version.yml index 72a536e..2cb16be 100644 --- a/.github/workflows/build-version.yml +++ b/.github/workflows/build-version.yml @@ -11,50 +11,91 @@ on: description: "Version tag to apply to Docker image (e.g. 3.14, 3.14j)" required: true type: string + pyenv_version: + description: "The pyenv version to use when building the builder-base stage" + required: true + type: string jobs: build: - runs-on: ubuntu-latest - name: Build Python ${{ inputs.version }} + name: Build Python ${{ inputs.version }} (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} - concurrency: - group: build-python-${{ inputs.version }} - cancel-in-progress: true + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v7 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to Github Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Get Git commit short SHA - id: git-sha - run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Sanitize platform name + id: platform + run: echo "name=$(echo '${{ matrix.platform }}' | tr '/' '-')" >> $GITHUB_OUTPUT - - name: Build and push Docker image - uses: docker/build-push-action@v6 + - name: Build and push platform image + uses: docker/build-push-action@v7 with: context: . file: ./Dockerfile + target: python-builder + platforms: ${{ matrix.platform }} push: ${{ github.ref == 'refs/heads/main' }} cache-to: type=inline - cache-from: type=registry,ref=ghcr.io/python-discord/python-builds:${{ inputs.tag }} + cache-from: | + type=registry,ref=ghcr.io/python-discord/python-builds:builder-base + type=registry,ref=ghcr.io/python-discord/python-builds:${{ inputs.tag }}-${{ steps.platform.outputs.name }} labels: | org.opencontainers.image.source=https://github.com/python-discord/python-builds org.opencontainers.image.description="Python ${{ inputs.version }} Docker image maintained by Python Discord." org.opencontainers.image.licenses=MIT - outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=Python ${{ inputs.version }} Docker image maintained by Python Discord. tags: | - ghcr.io/python-discord/python-builds:${{ inputs.tag }} - ghcr.io/python-discord/python-builds:${{ inputs.version }} - ghcr.io/python-discord/python-builds:${{ inputs.tag }}-${{ steps.git-sha.outputs.sha }} + ghcr.io/python-discord/python-builds:${{ inputs.tag }}-${{ steps.platform.outputs.name }} build-args: | PYTHON_VERSION=${{ inputs.version }} + PYENV_VERSION=${{ inputs.pyenv_version }} + + merge: + name: Merge ${{ inputs.version }} manifests + needs: build + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + + steps: + - name: Get Git commit short SHA + id: git-sha + run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Login to Github Container Registry + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create and push multi-platform manifest + run: | + docker buildx imagetools create \ + --annotation "index:org.opencontainers.image.source=https://github.com/python-discord/python-builds" \ + --annotation "index:org.opencontainers.image.description=Python ${{ inputs.version }} Docker image maintained by Python Discord." \ + --annotation "index:org.opencontainers.image.licenses=MIT" \ + --tag ghcr.io/python-discord/python-builds:${{ inputs.tag }} \ + --tag ghcr.io/python-discord/python-builds:${{ inputs.version }} \ + --tag ghcr.io/python-discord/python-builds:${{ inputs.tag }}-${{ steps.git-sha.outputs.sha }} \ + ghcr.io/python-discord/python-builds:${{ inputs.tag }}-linux-amd64 \ + ghcr.io/python-discord/python-builds:${{ inputs.tag }}-linux-arm64 diff --git a/Dockerfile b/Dockerfile index df5d11f..1c120f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,32 @@ -FROM ghcr.io/python-discord/python-builds:builder-base AS python-builder - LABEL org.opencontainers.image.authors="Joe Banks " +FROM buildpack-deps:bookworm AS builder-base +LABEL org.opencontainers.image.authors="Joe Banks , Chris Lovering " + +ARG PYENV_VERSION="v2.6.11" + +RUN apt-get -y update \ + && apt-get install -y --no-install-recommends \ + libxmlsec1-dev \ + tk-dev \ + lsb-release \ + software-properties-common \ + gnupg \ + && rm -rf /var/lib/apt/lists/* + +# Following guidance from https://github.com/python/cpython/blob/main/Tools/jit/README.md +RUN curl -o /tmp/llvm.sh https://apt.llvm.org/llvm.sh \ + && chmod +x /tmp/llvm.sh \ + && /tmp/llvm.sh 19 \ + && rm /tmp/llvm.sh + +ENV PYENV_ROOT=/pyenv \ + PYTHON_CONFIGURE_OPTS='--disable-test-modules --enable-optimizations \ + --with-lto --without-ensurepip' + +RUN git clone -b ${PYENV_VERSION} --depth 1 https://github.com/pyenv/pyenv.git $PYENV_ROOT + +COPY --link scripts scripts + +FROM builder-base AS python-builder ARG PYTHON_VERSION diff --git a/Dockerfile.base b/Dockerfile.base deleted file mode 100644 index 199612d..0000000 --- a/Dockerfile.base +++ /dev/null @@ -1,26 +0,0 @@ -FROM buildpack-deps:bookworm - LABEL org.opencontainers.image.authors="Joe Banks " -ARG PYENV_VERSION="v2.6.11" - -RUN apt-get -y update \ - && apt-get install -y --no-install-recommends \ - libxmlsec1-dev \ - tk-dev \ - lsb-release \ - software-properties-common \ - gnupg \ - && rm -rf /var/lib/apt/lists/* - -# Following guidance from https://github.com/python/cpython/blob/main/Tools/jit/README.md -RUN curl -o /tmp/llvm.sh https://apt.llvm.org/llvm.sh \ - && chmod +x /tmp/llvm.sh \ - && /tmp/llvm.sh 19 \ - && rm /tmp/llvm.sh - -ENV PYENV_ROOT=/pyenv \ - PYTHON_CONFIGURE_OPTS='--disable-test-modules --enable-optimizations \ - --with-lto --without-ensurepip' - -RUN git clone -b ${PYENV_VERSION} --depth 1 https://github.com/pyenv/pyenv.git $PYENV_ROOT - -COPY --link scripts scripts diff --git a/scripts/find_version.py b/scripts/find_version.py index b6f97b4..2457eea 100644 --- a/scripts/find_version.py +++ b/scripts/find_version.py @@ -1,6 +1,7 @@ from pathlib import Path import tomllib import json +import os def resolve_version(friendly_version: str) -> str: """Resolve a friendly version like '3.10t' to an exact version like '3.10.12'.""" @@ -9,7 +10,8 @@ def resolve_version(friendly_version: str) -> str: is_jit = friendly_version.endswith('j') base_version = friendly_version.rstrip('jt') - pyenv_versions = Path("/pyenv/plugins/python-build/share/python-build") + pyenv_root = Path(os.environ.get("PYENV_ROOT", "/pyenv")) + pyenv_versions = pyenv_root / "plugins/python-build/share/python-build" matching_versions = [] @@ -38,7 +40,7 @@ def resolve_version(friendly_version: str) -> str: return latest_version if __name__ == "__main__": - versions_toml_path = Path("/") / "versions.toml" + versions_toml_path = Path(os.environ.get("VERSIONS_TOML", "/versions.toml")) if not versions_toml_path.exists(): raise FileNotFoundError(f"Could not find versions.toml at expected path: {versions_toml_path}") diff --git a/versions.toml b/versions.toml index 10d2b52..0f5a430 100644 --- a/versions.toml +++ b/versions.toml @@ -16,7 +16,6 @@ [config] versions = [ - "3.14", "3.14t", "3.14j", "3.13",