Add arm64 support to CI Docker builds

.github/workflows/build-root.yml

@@ -17,68 +17,77 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       pyenv_version: ${{ steps.get_pyenv_version.outputs.pyenv_version }}
+      resolved_versions: ${{ steps.resolve_versions.outputs.resolved_versions }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
+
+      - name: Checkout pyenv
+        uses: actions/checkout@v7
         with:
           repository: pyenv/pyenv
           fetch-depth: 0
           fetch-tags: true
+          path: pyenv
 
       - name: Get latest pyenv version
         id: get_pyenv_version
         run: |
+          cd pyenv
           echo "pyenv_version=$(git describe --abbrev=0 --tags)" >> $GITHUB_OUTPUT
 
+      - name: Resolve Python versions
+        id: resolve_versions
+        run: |
+          resolved_versions=$(PYENV_ROOT=./pyenv VERSIONS_TOML=./versions.toml python3 scripts/find_version.py)
+          # JSON array with objects containing "tag" and "version" keys
+          echo "resolved_versions=$resolved_versions" >> $GITHUB_OUTPUT
+
   build-base:
     name: Build Python Builder Base
     needs: pyenv-version
-    outputs:
-      resolved_versions: ${{ steps.resolve_versions.outputs.resolved_versions }}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Login to Github Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push Python builder base
-        uses: docker/build-push-action@v6
+      - name: Build and push Python builder base (multi-platform)
+        uses: docker/build-push-action@v7
         with:
           context: .
-          file: ./Dockerfile.base
+          file: ./Dockerfile
+          target: builder-base
+          platforms: linux/amd64,linux/arm64
           push: ${{ github.ref == 'refs/heads/main' }}
           cache-to: type=inline
-          load: true
           cache-from: type=registry,ref=ghcr.io/python-discord/python-builds:builder-base
           build-args: |
             PYENV_VERSION=${{ needs.pyenv-version.outputs.pyenv_version }}
           tags: |
             ghcr.io/python-discord/python-builds:builder-base
             ghcr.io/python-discord/python-builds:builder-base-${{ needs.pyenv-version.outputs.pyenv_version }}
 
-      - name: Run Docker container to resolve versions
-        id: resolve_versions
-        run: |
-          resolved_versions=$(docker run --mount type=bind,src=./versions.toml,dst=/versions.toml --rm ghcr.io/python-discord/python-builds:builder-base python3 /scripts/find_version.py)
-          # JSON array with objects containing "tag" and "version" keys
-          echo "resolved_versions=$resolved_versions" >> $GITHUB_OUTPUT
-
   build-versions:
     name: Build Python Versions
-    needs: build-base
+    needs: [pyenv-version, build-base]
     strategy:
       matrix:
-        version_info: ${{ fromJson(needs.build-base.outputs.resolved_versions) }}
+        version_info: ${{ fromJson(needs.pyenv-version.outputs.resolved_versions) }}
     uses: ./.github/workflows/build-version.yml
     with:
       version: ${{ matrix.version_info.version }}
       tag: ${{ matrix.version_info.tag }}
+      pyenv_version: ${{ needs.pyenv-version.outputs.pyenv_version }}

.github/workflows/build-version.yml

@@ -11,50 +11,91 @@ on:
         description: "Version tag to apply to Docker image (e.g. 3.14, 3.14j)"
         required: true
         type: string
+      pyenv_version:
+        description: "The pyenv version to use when building the builder-base stage"
+        required: true
+        type: string
 
 jobs:
   build:
-    runs-on: ubuntu-latest
-    name: Build Python ${{ inputs.version }}
+    name: Build Python ${{ inputs.version }} (${{ matrix.platform }})
+    runs-on: ${{ matrix.runner }}
 
-    concurrency:
-      group: build-python-${{ inputs.version }}
-      cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            runner: ubuntu-latest
+          - platform: linux/arm64
+            runner: ubuntu-24.04-arm
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5
+        uses: actions/checkout@v7
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v4
 
       - name: Login to Github Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Get Git commit short SHA
-        id: git-sha
-        run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Sanitize platform name
+        id: platform
+        run: echo "name=$(echo '${{ matrix.platform }}' | tr '/' '-')" >> $GITHUB_OUTPUT
 
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v6
+      - name: Build and push platform image
+        uses: docker/build-push-action@v7
         with:
           context: .
           file: ./Dockerfile
+          target: python-builder
+          platforms: ${{ matrix.platform }}
           push: ${{ github.ref == 'refs/heads/main' }}
           cache-to: type=inline
-          cache-from: type=registry,ref=ghcr.io/python-discord/python-builds:${{ inputs.tag }}
+          cache-from: |
+            type=registry,ref=ghcr.io/python-discord/python-builds:builder-base
+            type=registry,ref=ghcr.io/python-discord/python-builds:${{ inputs.tag }}-${{ steps.platform.outputs.name }}
           labels: |
             org.opencontainers.image.source=https://github.com/python-discord/python-builds
             org.opencontainers.image.description="Python ${{ inputs.version }} Docker image maintained by Python Discord."
             org.opencontainers.image.licenses=MIT
-          outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=Python ${{ inputs.version }} Docker image maintained by Python Discord.
           tags: |
-            ghcr.io/python-discord/python-builds:${{ inputs.tag }}
-            ghcr.io/python-discord/python-builds:${{ inputs.version }}
-            ghcr.io/python-discord/python-builds:${{ inputs.tag }}-${{ steps.git-sha.outputs.sha }}
+            ghcr.io/python-discord/python-builds:${{ inputs.tag }}-${{ steps.platform.outputs.name }}
           build-args: |
             PYTHON_VERSION=${{ inputs.version }}
+            PYENV_VERSION=${{ inputs.pyenv_version }}
+
+  merge:
+    name: Merge ${{ inputs.version }} manifests
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
+
+    steps:
+      - name: Get Git commit short SHA
+        id: git-sha
+        run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Login to Github Container Registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push multi-platform manifest
+        run: |
+          docker buildx imagetools create \
+            --annotation "index:org.opencontainers.image.source=https://github.com/python-discord/python-builds" \
+            --annotation "index:org.opencontainers.image.description=Python ${{ inputs.version }} Docker image maintained by Python Discord." \
+            --annotation "index:org.opencontainers.image.licenses=MIT" \
+            --tag ghcr.io/python-discord/python-builds:${{ inputs.tag }} \
+            --tag ghcr.io/python-discord/python-builds:${{ inputs.version }} \
+            --tag ghcr.io/python-discord/python-builds:${{ inputs.tag }}-${{ steps.git-sha.outputs.sha }} \
+            ghcr.io/python-discord/python-builds:${{ inputs.tag }}-linux-amd64 \
+            ghcr.io/python-discord/python-builds:${{ inputs.tag }}-linux-arm64

Dockerfile

@@ -1,5 +1,32 @@
-FROM ghcr.io/python-discord/python-builds:builder-base AS python-builder
-    LABEL org.opencontainers.image.authors="Joe Banks <joe@owlcorp.uk>"
+FROM buildpack-deps:bookworm AS builder-base
+LABEL org.opencontainers.image.authors="Joe Banks <joe@owlcorp.uk>, Chris Lovering <cj@owlcorp.uk>"
+
+ARG PYENV_VERSION="v2.6.11"
+
+RUN apt-get -y update \
+    && apt-get install -y --no-install-recommends \
+        libxmlsec1-dev \
+        tk-dev \
+        lsb-release \
+        software-properties-common \
+        gnupg \
+    && rm -rf /var/lib/apt/lists/*
+
+# Following guidance from https://github.com/python/cpython/blob/main/Tools/jit/README.md
+RUN curl -o /tmp/llvm.sh https://apt.llvm.org/llvm.sh \
+    && chmod +x /tmp/llvm.sh \
+    && /tmp/llvm.sh 19 \
+    && rm /tmp/llvm.sh
+
+ENV PYENV_ROOT=/pyenv \
+    PYTHON_CONFIGURE_OPTS='--disable-test-modules --enable-optimizations \
+        --with-lto --without-ensurepip'
+
+RUN git clone -b ${PYENV_VERSION} --depth 1 https://github.com/pyenv/pyenv.git $PYENV_ROOT
+
+COPY --link scripts scripts
+
+FROM builder-base AS python-builder
 
 ARG PYTHON_VERSION
 

scripts/find_version.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 import tomllib
 import json
+import os
 
 def resolve_version(friendly_version: str) -> str:
     """Resolve a friendly version like '3.10t' to an exact version like '3.10.12'."""
@@ -9,7 +10,8 @@ def resolve_version(friendly_version: str) -> str:
     is_jit = friendly_version.endswith('j')
     base_version = friendly_version.rstrip('jt')
 
-    pyenv_versions = Path("/pyenv/plugins/python-build/share/python-build")
+    pyenv_root = Path(os.environ.get("PYENV_ROOT", "/pyenv"))
+    pyenv_versions = pyenv_root / "plugins/python-build/share/python-build"
 
     matching_versions = []
 
@@ -38,7 +40,7 @@ def resolve_version(friendly_version: str) -> str:
     return latest_version
 
 if __name__ == "__main__":
-    versions_toml_path = Path("/") / "versions.toml"
+    versions_toml_path = Path(os.environ.get("VERSIONS_TOML", "/versions.toml"))
 
     if not versions_toml_path.exists():
         raise FileNotFoundError(f"Could not find versions.toml at expected path: {versions_toml_path}")

versions.toml

@@ -16,7 +16,6 @@
 
 [config]
 versions = [
-    "3.14",
     "3.14t",
     "3.14j",
     "3.13",