From 8df2a8ff0593dc2cc93a348e7449b2d199730cd5 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Tue, 28 Jul 2026 10:02:16 +0100 Subject: [PATCH 1/2] ci: mirror Jetson Python wheels Keep the CUDA aarch64 wheel subset in GHCR and serve it as a local PEP 503 index during L4T backend builds, preserving last-known-good packages through upstream outages. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe --- .agents/ci-caching.md | 14 ++ .github/jetson-wheels.json | 25 ++ .github/workflows/backend_build.yml | 37 +++ .github/workflows/jetson-wheels.yml | 131 ++++++++++ backend/Dockerfile.python | 11 +- backend/python/common/libbackend.sh | 73 +++++- backend/python/common/pypi_mirror_server.py | 226 ++++++++++++++++++ .../python/common/pypi_mirror_server_test.py | 100 ++++++++ scripts/jetson-wheels-sync.py | 178 ++++++++++++++ 9 files changed, 792 insertions(+), 3 deletions(-) create mode 100644 .github/jetson-wheels.json create mode 100644 .github/workflows/jetson-wheels.yml create mode 100644 backend/python/common/pypi_mirror_server.py create mode 100644 backend/python/common/pypi_mirror_server_test.py create mode 100644 scripts/jetson-wheels-sync.py diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 1aa9c54ac3d0..645f54f06e2c 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -231,6 +231,20 @@ This applies only to `Dockerfile.python` because: Bump the format to daily (`+%Y-%m-%d`) or hourly (`+%Y-%m-%d-%H`) for faster refreshes. For one-shot rebuilds without changing the schedule, append a marker to the tag-suffix in the matrix or temporarily delete that backend's cache tag in quay. +## The jetson wheels mirror (l4t builds) + +The `requirements-l4t12.txt` / `-l4t13.txt` files pull CUDA aarch64 torch wheels from `pypi.jetson-ai-lab.io` via `--extra-index-url`. That index has a history of multi-hour 502 outages, and a 502 on **any** project page aborts the whole uv resolution — uv consults every configured index for every requirement, so even PyPI-hosted packages die with it. To keep l4t builds green through outages, CI serves those wheels from a mirror it controls: + +- **Storage**: `ghcr.io/mudler/localai/jetson-wheels:{jp6-cu129,jp7-cu130}` — scratch OCI images holding the wheel subset, laid out like the upstream index (`/jp6/cu129/torch/`). +- **Sync**: `.github/workflows/jetson-wheels.yml` (Saturdays 03:00 UTC, ahead of the weekly `DEPS_REFRESH` re-resolve; also `workflow_dispatch` and master pushes touching its inputs) runs `scripts/jetson-wheels-sync.py` against the package list in `.github/jetson-wheels.json`. During an upstream outage the sync keeps the last-known-good wheels and exits green. +- **Consumption**: `backend_build.yml` resolves the matching tag for `build-type: l4t` entries (cuda 12 → `jp6-cu129`, 13 → `jp7-cu130`) and passes it as the `JETSON_WHEELS_IMAGE` build-arg; `Dockerfile.python` bind-mounts it at `/jetson-wheels`; `installRequirements` in `backend/python/common/libbackend.sh` serves that directory on localhost as a PEP 503 index (`backend/python/common/pypi_mirror_server.py`) and rewrites the jetson index host in the requirements files to it. The local index 404s for anything it doesn't carry, which uv follows up on PyPI — only the jetson-built wheels resolve locally. +- **Fallbacks**: if the mirror tag doesn't exist (bootstrap) `backend_build.yml` passes `scratch`, the mount is empty, and the build talks to the upstream index exactly as before. Builds outside CI (local, real Jetsons) never set `JETSON_WHEELS_IMAGE` and are unaffected. +- **Cache interaction**: the bind mount's content is part of the `RUN ... make` layer's BuildKit hash, so a refreshed wheels image invalidates the install layer on the next build — no extra cache-buster needed. + +**Extending the package list**: a package a build needs from the jetson index but missing from `.github/jetson-wheels.json` resolves from PyPI instead — for compiled CUDA packages that silently means a CPU build. When adding an l4t backend with new compiled deps, add them to the list and dispatch `jetson-wheels.yml`. + +**Bootstrap** (one-time): `gh workflow run jetson-wheels.yml --ref master`, then make the `jetson-wheels` ghcr package public so anonymous pulls work (Settings → Packages). + ## ccache for C++ backend builds `Dockerfile.{llama-cpp,ik-llama-cpp,turboquant}` declare a BuildKit cache mount on `/root/.ccache`: diff --git a/.github/jetson-wheels.json b/.github/jetson-wheels.json new file mode 100644 index 000000000000..883bf143fedb --- /dev/null +++ b/.github/jetson-wheels.json @@ -0,0 +1,25 @@ +{ + "upstream": "https://pypi.jetson-ai-lab.io", + "indexes": { + "jp6/cu129": [ + "torch", + "torchvision", + "torchaudio", + "torchcodec", + "torchao", + "bitsandbytes", + "onnxruntime", + "ctranslate2" + ], + "jp7/cu130": [ + "torch", + "torchvision", + "torchaudio", + "torchcodec", + "torchao", + "bitsandbytes", + "onnxruntime", + "ctranslate2" + ] + } +} diff --git a/.github/workflows/backend_build.yml b/.github/workflows/backend_build.yml index 05d50cf821c2..c4358d8ef462 100644 --- a/.github/workflows/backend_build.yml +++ b/.github/workflows/backend_build.yml @@ -181,6 +181,41 @@ jobs: id: deps_refresh run: echo "key=$(date -u +%Y-W%V)" >> "$GITHUB_OUTPUT" + - name: Login to ghcr.io (jetson wheels mirror) + if: inputs.build-type == 'l4t' + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + # l4t builds pull their CUDA aarch64 torch wheels from + # pypi.jetson-ai-lab.io, which has a history of multi-hour 502 outages + # that fail every l4t job. jetson-wheels.yml mirrors those wheels into + # ghcr weekly; here we hand the mirror image to Dockerfile.python, + # which serves it as a local package index during pip install (see + # installRequirements in backend/python/common/libbackend.sh). Falls + # back to scratch — i.e. building straight against the upstream index — + # when the mirror tag doesn't exist yet, so the mirror can bootstrap + # without a chicken-and-egg failure. + - name: Resolve jetson wheels mirror image + id: jetson_wheels + if: inputs.build-type == 'l4t' + run: | + repo="ghcr.io/$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')/jetson-wheels" + case "${{ inputs.cuda-major-version }}" in + 12) tag="jp6-cu129" ;; + 13) tag="jp7-cu130" ;; + *) tag="" ;; + esac + img="" + if [ -n "$tag" ] && docker buildx imagetools inspect "$repo:$tag" >/dev/null 2>&1; then + img="$repo:$tag" + else + echo "jetson wheels image $repo:$tag not found; building against the upstream index" + fi + echo "image=$img" >> "$GITHUB_OUTPUT" + - name: Build and push by digest id: build uses: docker/build-push-action@v7 @@ -201,6 +236,7 @@ jobs: DEPS_REFRESH=${{ steps.deps_refresh.outputs.key }} BUILDER_BASE_IMAGE=${{ inputs.builder-base-image }} BUILDER_TARGET=${{ inputs.builder-base-image != '' && 'builder-prebuilt' || 'builder-fromsource' }} + JETSON_WHEELS_IMAGE=${{ steps.jetson_wheels.outputs.image || 'scratch' }} context: ${{ inputs.context }} file: ${{ inputs.dockerfile }} cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} @@ -273,6 +309,7 @@ jobs: DEPS_REFRESH=${{ steps.deps_refresh.outputs.key }} BUILDER_BASE_IMAGE=${{ inputs.builder-base-image }} BUILDER_TARGET=${{ inputs.builder-base-image != '' && 'builder-prebuilt' || 'builder-fromsource' }} + JETSON_WHEELS_IMAGE=${{ steps.jetson_wheels.outputs.image || 'scratch' }} context: ${{ inputs.context }} file: ${{ inputs.dockerfile }} cache-from: type=registry,ref=quay.io/go-skynet/ci-cache:cache${{ inputs.tag-suffix }}-${{ inputs.platform-tag }} diff --git a/.github/workflows/jetson-wheels.yml b/.github/workflows/jetson-wheels.yml new file mode 100644 index 000000000000..a943418a8464 --- /dev/null +++ b/.github/workflows/jetson-wheels.yml @@ -0,0 +1,131 @@ +--- +name: 'sync jetson wheels mirror' + +# Mirrors the CUDA aarch64 wheels our l4t backends need from +# pypi.jetson-ai-lab.io into scratch OCI images on ghcr +# (ghcr.io/mudler/localai/jetson-wheels:, one tag per JetPack index). +# backend_build.yml hands the matching tag to Dockerfile.python, which +# bind-mounts it and serves it as a local package index during pip install +# (see installRequirements in backend/python/common/libbackend.sh), so the +# upstream index's recurring multi-hour 502 outages can no longer fail l4t +# builds. +# +# The package subset lives in .github/jetson-wheels.json. A package that a +# build needs from the jetson index but that is missing from that list will +# resolve from PyPI instead — for compiled CUDA packages that silently means +# a CPU build, so extend the list when adding an l4t backend with new +# compiled deps. +# +# When upstream is unreachable the sync keeps the previously mirrored wheels +# and exits green — the mirror serves last-known-good through outages. It +# only fails when upstream is down and the tag has never been published +# (bootstrap during an outage: nothing to serve yet). +# +# Triggers: +# - schedule (Saturdays 03:00 UTC) — refreshes ahead of base-images.yml +# (Saturdays 05:00 UTC) and the backend.yml weekly cron (Sundays), whose +# DEPS_REFRESH cache-bust re-resolves the python deps. +# - workflow_dispatch — manual one-off sync; also the bootstrap run: +# gh workflow run jetson-wheels.yml --ref master +# - push to master touching the config, the sync script, or this workflow. + +on: + schedule: + - cron: '0 3 * * 6' + workflow_dispatch: + push: + branches: [master] + paths: + - '.github/jetson-wheels.json' + - 'scripts/jetson-wheels-sync.py' + - '.github/workflows/jetson-wheels.yml' + +permissions: + contents: read + packages: write + +concurrency: + group: jetson-wheels-${{ github.repository }} + cancel-in-progress: false + +jobs: + sync: + if: github.repository == 'mudler/LocalAI' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - index: 'jp6/cu129' + tag: 'jp6-cu129' + - index: 'jp7/cu130' + tag: 'jp7-cu130' + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@master + + - name: Login to ghcr.io + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Compute image name + id: image + run: | + repo="ghcr.io/$(echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')/jetson-wheels" + echo "ref=${repo}:${{ matrix.tag }}" >> "$GITHUB_OUTPUT" + + # Seed the working dir with the current mirror contents so the sync is + # incremental and an upstream outage keeps last-known-good wheels. + - name: Pull current mirror contents + run: | + mkdir -p wheels + # The image is declared linux/arm64 (its only consumers are arm64 + # l4t builds); pulling on this amd64 runner needs the explicit + # platform. The content is just wheel files — never executed here. + if docker pull --platform linux/arm64 "${{ steps.image.outputs.ref }}"; then + # scratch images have no command; docker create still needs one, + # but the container is never started so any path works. + cid="$(docker create "${{ steps.image.outputs.ref }}" /noop)" + docker export "${cid}" | tar -x -C wheels + docker rm "${cid}" + find wheels -name '*.whl' | sed 's/^/ existing: /' + else + echo "no existing mirror image (bootstrap run)" + fi + + - name: Sync from upstream + id: sync + run: | + python3 scripts/jetson-wheels-sync.py \ + --config .github/jetson-wheels.json \ + --index '${{ matrix.index }}' \ + --dest wheels \ + --changed-file /tmp/jetson-wheels-changed + if [ -f /tmp/jetson-wheels-changed ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Push mirror image + if: steps.sync.outputs.changed == 'true' + run: | + cat > Dockerfile.jetson-wheels <<'EOF' + FROM scratch + COPY wheels/ / + EOF + # linux/arm64 because the consumers (l4t builds in + # backend_build.yml) build for arm64 and BuildKit refuses a + # platform-mismatched FROM; COPY-only, so no emulation is needed. + # provenance=false keeps the pushed ref a plain single manifest + # instead of an OCI index wrapping an attestation. + docker buildx build --push \ + --platform linux/arm64 \ + --provenance=false \ + -f Dockerfile.jetson-wheels \ + -t "${{ steps.image.outputs.ref }}" \ + . diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 2522a6f56be0..511a4e7304ee 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -1,6 +1,14 @@ ARG BASE_IMAGE=ubuntu:24.04 ARG APT_MIRROR="" ARG APT_PORTS_MIRROR="" +# CI mirror of the CUDA aarch64 wheels from pypi.jetson-ai-lab.io, kept warm +# by .github/workflows/jetson-wheels.yml so l4t builds survive the upstream +# index's recurring multi-hour outages. The default (scratch) mounts an empty +# directory, which makes installRequirements fall through to the upstream +# index unchanged — local and Jetson-native builds are unaffected. +ARG JETSON_WHEELS_IMAGE=scratch + +FROM ${JETSON_WHEELS_IMAGE} AS jetson-wheels FROM ${BASE_IMAGE} AS builder ARG BACKEND=rerankers @@ -222,7 +230,8 @@ ENV FROM_SOURCE=${FROM_SOURCE} # and picks up newer wheels from PyPI / nightly indexes. ARG DEPS_REFRESH=initial -RUN cd /${BACKEND} && PORTABLE_PYTHON=true make +RUN --mount=type=bind,from=jetson-wheels,target=/jetson-wheels \ + cd /${BACKEND} && PORTABLE_PYTHON=true JETSON_WHEELS_DIR=/jetson-wheels make # Package GPU libraries into the backend's lib directory. # diff --git a/backend/python/common/libbackend.sh b/backend/python/common/libbackend.sh index 14172decf506..b8ec5c966a6d 100644 --- a/backend/python/common/libbackend.sh +++ b/backend/python/common/libbackend.sh @@ -475,6 +475,62 @@ function runProtogen() { } +# When JETSON_WHEELS_DIR points at a directory of wheels mirrored from +# pypi.jetson-ai-lab.io (CI bind-mounts the jetson-wheels OCI image there — +# see backend/Dockerfile.python and .github/workflows/jetson-wheels.yml), +# installRequirements serves it on localhost as a PEP 503 index and swaps the +# jetson index host in the requirements files for the local one. +# +# The upstream index has a history of multi-hour 502 outages, and a 502 on +# any project page aborts the whole uv resolution — uv consults every +# configured index for every requirement, so even PyPI-hosted packages die +# with it. The local index instead 404s for anything it doesn't carry, which +# resolvers cleanly follow up on PyPI; only the jetson-built wheels (torch +# and friends) resolve locally. When JETSON_WHEELS_DIR is unset — the +# default, e.g. building on a real Jetson — nothing changes and the upstream +# index is used as written in the requirements files. +JETSON_PYPI_HOST="pypi.jetson-ai-lab.io" +_JETSON_MIRROR_PID="" +_JETSON_MIRROR_URL="" + +function _stopJetsonMirror() { + if [ -n "${_JETSON_MIRROR_PID}" ]; then + kill "${_JETSON_MIRROR_PID}" 2>/dev/null || true + _JETSON_MIRROR_PID="" + _JETSON_MIRROR_URL="" + fi +} + +function _startJetsonMirror() { + local script_dir port_file port tries + # An empty dir is the JETSON_WHEELS_IMAGE=scratch default in + # Dockerfile.python: no mirror was provided, use upstream as-is. + if [ -z "$(find "${JETSON_WHEELS_DIR}" -name '*.whl' -print -quit 2>/dev/null)" ]; then + echo "jetson wheels dir ${JETSON_WHEELS_DIR} has no wheels, using upstream ${JETSON_PYPI_HOST}" + return 0 + fi + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + port_file="$(mktemp)" + rm -f "${port_file}" + python3 "${script_dir}/pypi_mirror_server.py" --root "${JETSON_WHEELS_DIR}" --port-file "${port_file}" & + _JETSON_MIRROR_PID=$! + trap _stopJetsonMirror EXIT + tries=0 + until [ -s "${port_file}" ]; do + tries=$((tries + 1)) + if [ ${tries} -gt 50 ] || ! kill -0 "${_JETSON_MIRROR_PID}" 2>/dev/null; then + echo "WARNING: local jetson wheel mirror failed to start, using upstream ${JETSON_PYPI_HOST}" + _stopJetsonMirror + return 0 + fi + sleep 0.2 + done + port="$(cat "${port_file}")" + rm -f "${port_file}" + _JETSON_MIRROR_URL="http://127.0.0.1:${port}" + echo "serving jetson wheels from ${JETSON_WHEELS_DIR} at ${_JETSON_MIRROR_URL}" +} + # installRequirements looks for several requirements files and if they exist runs the install for them in order # # - requirements-install.txt @@ -520,18 +576,31 @@ function installRequirements() { export C_INCLUDE_PATH="${C_INCLUDE_PATH:-}:$(_portable_dir)/include/python${PYTHON_VERSION}" fi + if [ -n "${JETSON_WHEELS_DIR:-}" ] && [ -d "${JETSON_WHEELS_DIR}" ]; then + _startJetsonMirror + fi + + local installFile for reqFile in ${requirementFiles[@]}; do if [ -f "${reqFile}" ]; then + installFile="${reqFile}" + if [ -n "${_JETSON_MIRROR_URL}" ] && grep -q "${JETSON_PYPI_HOST}" "${reqFile}"; then + installFile="$(mktemp)" + sed "s,https://${JETSON_PYPI_HOST},${_JETSON_MIRROR_URL},g" "${reqFile}" > "${installFile}" + echo "rewrote ${JETSON_PYPI_HOST} in ${reqFile} to the local wheel mirror (${installFile})" + fi echo "starting requirements install for ${reqFile}" if [ "x${USE_PIP}" == "xtrue" ]; then - pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}" + pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${installFile}" else - uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}" + uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${installFile}" fi echo "finished requirements install for ${reqFile}" fi done + _stopJetsonMirror + runProtogen } diff --git a/backend/python/common/pypi_mirror_server.py b/backend/python/common/pypi_mirror_server.py new file mode 100644 index 000000000000..394c3f0005dc --- /dev/null +++ b/backend/python/common/pypi_mirror_server.py @@ -0,0 +1,226 @@ +"""Ephemeral PEP 503 "simple" index over a local directory of wheels. + +Serves a directory tree laid out like a package index (e.g. +``/jp6/cu129/torch/torch-2.8.0-cp312-...whl``) as a standards-compliant +"simple" index on localhost, so uv/pip can resolve against it exactly as they +would against the real remote index — same per-project pages, same 404 +fall-through to PyPI for projects the mirror does not carry. + +This exists because pypi.jetson-ai-lab.io (the only source of CUDA-enabled +aarch64 torch wheels for JetPack) has a history of multi-hour 502 outages, +and an --extra-index-url that errors is fatal to the whole resolution: uv +consults every configured index for every requirement, so one 502 on any +project page kills the install even for projects hosted on PyPI. CI mirrors +the handful of jetson-only wheels into an OCI image, bind-mounts it into the +backend build, and libbackend.sh serves it with this script while rewriting +the index host in the requirements files to 127.0.0.1 (see +installRequirements in libbackend.sh). A 404 from this server is a clean +"not here" that resolvers follow up on PyPI; the upstream 502 never was. + +Standard library only — it runs inside every python backend's build +container, before any venv exists. + +Usage: + python3 pypi_mirror_server.py --root /jetson-wheels --port-file /tmp/port + +Run the tests standalone: + python3 -m unittest pypi_mirror_server_test +""" + +import argparse +import hashlib +import html +import os +import re +import sys +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# Extensions treated as distribution files: a directory containing at least +# one of these is a project page; any other directory is a sub-index listing. +DIST_SUFFIXES = (".whl", ".tar.gz", ".zip") + +_hash_cache = {} +_hash_lock = threading.Lock() + + +def normalize(name): + """PEP 503 project-name normalization.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def _file_sha256(path): + """sha256 of a file, cached on (path, mtime, size) — wheels are large.""" + st = os.stat(path) + key = (path, st.st_mtime_ns, st.st_size) + with _hash_lock: + cached = _hash_cache.get(key) + if cached: + return cached + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + value = digest.hexdigest() + with _hash_lock: + _hash_cache[key] = value + return value + + +def resolve_path(root, url_path): + """Map a URL path onto the tree under root, or None. + + Path segments match either exactly or via PEP 503 normalization + (resolvers request ``liquid-audio`` even if the directory on disk is + named ``liquid_audio``). Rejects any segment that would escape root. + """ + current = root + for segment in url_path.split("/"): + if segment in ("", "."): + continue + if segment == ".." or "/" in segment or "\\" in segment: + return None + candidate = os.path.join(current, segment) + if not os.path.exists(candidate): + try: + entries = os.listdir(current) + except (NotADirectoryError, FileNotFoundError): + return None + wanted = normalize(segment) + matches = [e for e in entries if normalize(e) == wanted] + if not matches: + return None + candidate = os.path.join(current, matches[0]) + current = candidate + return current + + +class SimpleIndexHandler(BaseHTTPRequestHandler): + root = None + protocol_version = "HTTP/1.1" + + def do_GET(self): + self._respond(head_only=False) + + def do_HEAD(self): + self._respond(head_only=True) + + def _respond(self, head_only): + url_path = urllib.parse.unquote(urllib.parse.urlsplit(self.path).path) + local = resolve_path(self.root, url_path) + if local is None: + self._send_error(404, "not found") + return + if os.path.isfile(local): + self._send_file(local, head_only) + return + # Relative hrefs on index pages resolve against the request URL, so + # directory URLs must end in "/" — redirect like real indexes do. + if not url_path.endswith("/"): + self.send_response(301) + self.send_header("Location", self.path + "/") + self.send_header("Content-Length", "0") + self.end_headers() + return + entries = sorted(os.listdir(local)) + files = [e for e in entries if e.endswith(DIST_SUFFIXES)] + if files: + body = self._project_page(local, files) + else: + dirs = [e for e in entries if os.path.isdir(os.path.join(local, e))] + body = self._listing_page(dirs) + self._send_html(body, head_only) + + def _project_page(self, project_dir, files): + anchors = [] + for name in files: + digest = _file_sha256(os.path.join(project_dir, name)) + anchors.append( + '%s
' + % (urllib.parse.quote(name), digest, html.escape(name)) + ) + return self._page(anchors) + + def _listing_page(self, dirs): + anchors = [ + '%s
' + % (urllib.parse.quote(normalize(d)), html.escape(normalize(d))) + for d in dirs + ] + return self._page(anchors) + + def _page(self, anchors): + return ( + "" + '' + "simple index\n" + + "\n".join(anchors) + + "\n" + ).encode() + + def _send_html(self, body, head_only): + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if not head_only: + self.wfile.write(body) + + def _send_file(self, path, head_only): + size = os.path.getsize(path) + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(size)) + self.end_headers() + if head_only: + return + with open(path, "rb") as f: + while True: + chunk = f.read(1 << 20) + if not chunk: + break + self.wfile.write(chunk) + + def _send_error(self, code, message): + body = message.encode() + self.send_response(code) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + sys.stderr.write("pypi-mirror: %s\n" % (format % args)) + + +def make_server(root, host="127.0.0.1", port=0): + handler = type("Handler", (SimpleIndexHandler,), {"root": os.path.abspath(root)}) + return ThreadingHTTPServer((host, port), handler) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", required=True, help="directory tree to serve") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=0, help="0 picks a free port") + parser.add_argument( + "--port-file", + help="write the bound port here once listening (readiness signal)", + ) + args = parser.parse_args() + + server = make_server(args.root, args.host, args.port) + port = server.server_address[1] + if args.port_file: + # Write-then-rename so a reader never sees a partially written port. + tmp = args.port_file + ".tmp" + with open(tmp, "w") as f: + f.write(str(port)) + os.replace(tmp, args.port_file) + sys.stderr.write("pypi-mirror: serving %s on %s:%d\n" % (args.root, args.host, port)) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/backend/python/common/pypi_mirror_server_test.py b/backend/python/common/pypi_mirror_server_test.py new file mode 100644 index 000000000000..5121b159d495 --- /dev/null +++ b/backend/python/common/pypi_mirror_server_test.py @@ -0,0 +1,100 @@ +"""Unit tests for the ephemeral PEP 503 index (pypi_mirror_server.py). + +Run standalone (Python standard library only, no backend venv needed): + python3 -m unittest pypi_mirror_server_test +""" + +import hashlib +import os +import shutil +import tempfile +import threading +import unittest +import urllib.error +import urllib.request + +from pypi_mirror_server import make_server, normalize, resolve_path + +WHEEL_BYTES = b"not a real wheel, but the server must serve it verbatim" + + +class TestHelpers(unittest.TestCase): + def test_normalize(self): + self.assertEqual(normalize("Liquid_Audio.Extra"), "liquid-audio-extra") + self.assertEqual(normalize("torch"), "torch") + + def test_resolve_rejects_traversal(self): + root = tempfile.mkdtemp() + try: + self.assertIsNone(resolve_path(root, "/../etc/passwd")) + self.assertIsNone(resolve_path(root, "/a/../../etc")) + finally: + shutil.rmtree(root) + + def test_resolve_normalized_segment(self): + root = tempfile.mkdtemp() + try: + os.makedirs(os.path.join(root, "jp6", "liquid_audio")) + found = resolve_path(root, "/jp6/liquid-audio/") + self.assertEqual(found, os.path.join(root, "jp6", "liquid_audio")) + finally: + shutil.rmtree(root) + + +class TestServer(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = tempfile.mkdtemp() + project = os.path.join(cls.root, "jp6", "cu129", "torch") + os.makedirs(project) + cls.wheel_name = "torch-2.8.0-cp312-cp312-linux_aarch64.whl" + with open(os.path.join(project, cls.wheel_name), "wb") as f: + f.write(WHEEL_BYTES) + cls.server = make_server(cls.root) + cls.base = "http://127.0.0.1:%d" % cls.server.server_address[1] + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + shutil.rmtree(cls.root) + + def _get(self, path): + with urllib.request.urlopen(self.base + path) as resp: + return resp.status, resp.read() + + def test_project_page_lists_wheel_with_hash(self): + status, body = self._get("/jp6/cu129/torch/") + self.assertEqual(status, 200) + digest = hashlib.sha256(WHEEL_BYTES).hexdigest() + self.assertIn( + ('' % (self.wheel_name, digest)).encode(), body + ) + + def test_index_listing_names_projects(self): + status, body = self._get("/jp6/cu129/") + self.assertEqual(status, 200) + self.assertIn(b'torch', body) + + def test_wheel_download_is_verbatim(self): + status, body = self._get("/jp6/cu129/torch/" + self.wheel_name) + self.assertEqual(status, 200) + self.assertEqual(body, WHEEL_BYTES) + + def test_unknown_project_is_404(self): + # 404 (not 5xx) matters: resolvers treat it as "not in this index" + # and fall back to PyPI, which is the whole point of the mirror. + with self.assertRaises(urllib.error.HTTPError) as ctx: + self._get("/jp6/cu129/liquid-audio/") + self.assertEqual(ctx.exception.code, 404) + + def test_directory_without_slash_redirects(self): + status, _ = self._get("/jp6/cu129/torch") + # urllib follows the 301; landing on the page proves the redirect + self.assertEqual(status, 200) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/jetson-wheels-sync.py b/scripts/jetson-wheels-sync.py new file mode 100644 index 000000000000..fd41c623e3ba --- /dev/null +++ b/scripts/jetson-wheels-sync.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Mirror the jetson-only wheel subset from pypi.jetson-ai-lab.io. + +Downloads the wheels for the packages listed in .github/jetson-wheels.json +(one list per JetPack index path) into a local directory laid out exactly +like the upstream index (///). The jetson-wheels +CI workflow publishes that directory as a scratch OCI image on ghcr, and +backend builds serve it as a local package index during pip install — see +pypi_mirror_server.py and installRequirements in +backend/python/common/libbackend.sh for the consuming side and the +motivation (recurring multi-hour upstream outages). + +The sync is additive-with-pruning against a *successfully fetched* project +page: files no longer listed upstream are removed, but a package whose page +cannot be fetched is left exactly as mirrored last time. When the whole +upstream is unreachable the existing mirror is kept as-is (exit 0) so a CI +run during an outage never destroys the last known-good wheels; it only +fails (exit 2) when upstream is down AND there is nothing mirrored yet, +i.e. the bootstrap run has nothing to publish. + +Standard library only. Usage: + python3 scripts/jetson-wheels-sync.py --config .github/jetson-wheels.json \ + --index jp6/cu129 --dest wheels [--changed-file /tmp/changed] +""" + +import argparse +import hashlib +import html.parser +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request + +DIST_SUFFIXES = (".whl", ".tar.gz", ".zip") +TIMEOUT = 120 + + +def normalize(name): + """PEP 503 project-name normalization.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +class _LinkParser(html.parser.HTMLParser): + def __init__(self): + super().__init__() + self.hrefs = [] + + def handle_starttag(self, tag, attrs): + if tag == "a": + for key, value in attrs: + if key == "href" and value: + self.hrefs.append(value) + + +def parse_links(page, base_url): + """Extract (filename, absolute_url, sha256|None) for each dist link.""" + parser = _LinkParser() + parser.feed(page) + links = [] + for href in parser.hrefs: + split = urllib.parse.urlsplit(href) + filename = os.path.basename(urllib.parse.unquote(split.path)) + if not filename.endswith(DIST_SUFFIXES): + continue + sha256 = None + if split.fragment.startswith("sha256="): + sha256 = split.fragment[len("sha256="):] + url = urllib.parse.urljoin(base_url, split._replace(fragment="").geturl()) + links.append((filename, url, sha256)) + return links + + +def _fetch(url): + request = urllib.request.Request(url, headers={"User-Agent": "localai-jetson-wheels-sync"}) + return urllib.request.urlopen(request, timeout=TIMEOUT) + + +def _download(url, dest_path, sha256): + digest = hashlib.sha256() + tmp = dest_path + ".tmp" + with _fetch(url) as resp, open(tmp, "wb") as out: + for chunk in iter(lambda: resp.read(1 << 20), b""): + digest.update(chunk) + out.write(chunk) + if sha256 and digest.hexdigest() != sha256: + os.unlink(tmp) + raise RuntimeError(f"sha256 mismatch for {url}: expected {sha256}, got {digest.hexdigest()}") + os.replace(tmp, dest_path) + + +def _has_wheels(dest): + for _, _, files in os.walk(dest): + if any(f.endswith(DIST_SUFFIXES) for f in files): + return True + return False + + +def sync_package(base_url, package, dest_dir): + """Returns (fetched_ok, changed).""" + page_url = urllib.parse.urljoin(base_url, normalize(package) + "/") + try: + with _fetch(page_url) as resp: + page = resp.read().decode("utf-8", "replace") + except urllib.error.HTTPError as err: + if err.code == 404: + # Upstream simply doesn't host this package for this index — + # normal (the config list is a superset across JetPack versions). + print(f" {package}: not hosted upstream (404), skipping") + return True, False + print(f" {package}: upstream error {err.code}, keeping mirrored files") + return False, False + except (urllib.error.URLError, TimeoutError, OSError) as err: + print(f" {package}: upstream unreachable ({err}), keeping mirrored files") + return False, False + + links = parse_links(page, page_url) + listed = {name for name, _, _ in links} + changed = False + os.makedirs(dest_dir, exist_ok=True) + for name, url, sha256 in links: + path = os.path.join(dest_dir, name) + if os.path.exists(path): + continue + print(f" {package}: downloading {name}") + _download(url, path, sha256) + changed = True + # Prune only against a page we actually fetched: upstream removing a + # wheel is tracked, an outage never empties the mirror. + for existing in os.listdir(dest_dir): + if existing.endswith(DIST_SUFFIXES) and existing not in listed: + print(f" {package}: pruning {existing} (no longer listed upstream)") + os.unlink(os.path.join(dest_dir, existing)) + changed = True + return True, changed + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True) + parser.add_argument("--index", required=True, help="index path, e.g. jp6/cu129") + parser.add_argument("--dest", required=True) + parser.add_argument("--upstream", help="override the config's upstream (for tests)") + parser.add_argument("--changed-file", help="created iff the mirror content changed") + args = parser.parse_args() + + with open(args.config) as f: + config = json.load(f) + packages = config["indexes"][args.index] + upstream = args.upstream or config["upstream"] + base_url = upstream.rstrip("/") + "/" + args.index.strip("/") + "/" + + print(f"syncing {args.index} from {base_url}: {', '.join(packages)}") + any_fetched = False + any_changed = False + for package in packages: + dest_dir = os.path.join(args.dest, args.index, normalize(package)) + fetched, changed = sync_package(base_url, package, dest_dir) + any_fetched = any_fetched or fetched + any_changed = any_changed or changed + + if not any_fetched: + if _has_wheels(os.path.join(args.dest, args.index)): + print("upstream unreachable; keeping existing mirror unchanged") + return 0 + print("upstream unreachable and nothing mirrored yet — nothing to publish") + return 2 + if any_changed and args.changed_file: + with open(args.changed_file, "w") as f: + f.write("changed\n") + print("sync complete" + (" (changes)" if any_changed else " (no changes)")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 752b518c9151bf05f5a086498c587e2a844b2528 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Wed, 29 Jul 2026 14:02:20 +0100 Subject: [PATCH 2/2] docs(agents): index the Jetson wheels mirror Mention the GHCR-hosted L4T wheel mirror in the CI caching guide summary so maintainers can find its outage and cache documentation. Assisted-by: Codex:gpt-5 Signed-off-by: Richard Palethorpe --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index dd2c79125a61..21c0953f3034 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ LocalAI follows the Linux kernel project's [guidelines for AI coding assistants] |------|-------------| | [.agents/ai-coding-assistants.md](.agents/ai-coding-assistants.md) | Policy for AI-assisted contributions — licensing, DCO, attribution | | [.agents/building-and-testing.md](.agents/building-and-testing.md) | Building the project, running tests, Docker builds for specific platforms | -| [.agents/ci-caching.md](.agents/ci-caching.md) | CI build cache layout (registry-backed BuildKit cache on quay.io/go-skynet/ci-cache, per-arch keys), `DEPS_REFRESH` weekly cache-buster for unpinned Python deps, prebuilt `base-grpc-*` images for llama.cpp variants, per-arch native + manifest-merge pattern, `setup-build-disk` `/mnt` relocation, path filter on master push, manual eviction | +| [.agents/ci-caching.md](.agents/ci-caching.md) | CI build cache layout (registry-backed BuildKit cache on quay.io/go-skynet/ci-cache, per-arch keys), `DEPS_REFRESH` weekly cache-buster for unpinned Python deps, jetson wheels mirror for l4t builds (ghcr-hosted, survives pypi.jetson-ai-lab.io outages), prebuilt `base-grpc-*` images for llama.cpp variants, per-arch native + manifest-merge pattern, `setup-build-disk` `/mnt` relocation, path filter on master push, manual eviction | | [.agents/adding-backends.md](.agents/adding-backends.md) | Adding a new backend (Python, Go, or C++) — full step-by-step checklist, including importer integration (the `/import-model` dropdown is server-driven from `GET /backends/known`) | | [.agents/coding-style.md](.agents/coding-style.md) | Code style, editorconfig, logging, documentation conventions | | [.agents/llama-cpp-backend.md](.agents/llama-cpp-backend.md) | Working on the llama.cpp backend — architecture, updating, tool call parsing |