From f93d54e7cb38692a52afeef9a9ec76747a8a3399 Mon Sep 17 00:00:00 2001 From: bvolpato Date: Tue, 4 Aug 2026 00:36:08 -0400 Subject: [PATCH 1/3] [CI] Publish GB200 aarch64 wheel artifacts Signed-off-by: bvolpato --- .../actions/build-pytorch-wheel/Dockerfile | 12 ++- .../workflows/attach-wheels-to-release.yml | 73 ++++++++++++++++++- build_tools/wheel_utils/Dockerfile.aarch | 5 +- 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/.github/actions/build-pytorch-wheel/Dockerfile b/.github/actions/build-pytorch-wheel/Dockerfile index 05ac20db0d..d3b0bd6f98 100644 --- a/.github/actions/build-pytorch-wheel/Dockerfile +++ b/.github/actions/build-pytorch-wheel/Dockerfile @@ -36,10 +36,18 @@ RUN CUDA_MAJOR_VERSION=$(echo $CUDA_VERSION | awk -F \. {'print $1'}) && \ dpkg -i cuda-keyring_1.1-1_all.deb && \ rm cuda-keyring_1.1-1_all.deb && \ apt-get update && \ - apt-get install -y cuda-toolkit-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} cudnn-cuda-$CUDA_MAJOR_VERSION libcudnn$CUDNN_MAJOR_VERSION-cuda-$CUDA_MAJOR_VERSION libnccl2 libnccl-dev cmake + apt-get install -y \ + cuda-compiler-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \ + cuda-libraries-dev-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \ + cuda-nvtx-${CUDA_MAJOR_VERSION}-${CUDA_MINOR_VERSION} \ + cudnn-cuda-$CUDA_MAJOR_VERSION \ + libcudnn$CUDNN_MAJOR_VERSION-cuda-$CUDA_MAJOR_VERSION \ + libnccl2 \ + libnccl-dev \ + cmake # Install PyTorch RUN export MATRIX_CUDA_VERSION=$(echo $CUDA_VERSION | awk -F \. {'print $1 $2'}) && \ export MATRIX_TORCH_VERSION=$(echo $TORCH_VERSION | awk -F \. {'print $1 "." $2'}) && \ - export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; versions = {'2.5': (118, 124), '2.6': (118, 126), '2.7': (118, 128), '2.8': (126, 129), '2.9': (126, 130)}; minv, maxv = versions[env['MATRIX_TORCH_VERSION']]; print(minv if int(env['MATRIX_CUDA_VERSION']) < 120 else maxv)") && \ + export TORCH_CUDA_VERSION=$(python -c "from os import environ as env; versions = {'2.5': (118, 124), '2.6': (118, 126), '2.7': (118, 128), '2.8': (126, 129), '2.9': (126, 130), '2.11': (130, 130)}; minv, maxv = versions[env['MATRIX_TORCH_VERSION']]; print(minv if int(env['MATRIX_CUDA_VERSION']) < 120 else maxv)") && \ pip install --no-cache-dir torch==${TORCH_VERSION} --index-url https://download.pytorch.org/whl/cu${TORCH_CUDA_VERSION} diff --git a/.github/workflows/attach-wheels-to-release.yml b/.github/workflows/attach-wheels-to-release.yml index fb56622d77..5568fa170f 100644 --- a/.github/workflows/attach-wheels-to-release.yml +++ b/.github/workflows/attach-wheels-to-release.yml @@ -32,11 +32,11 @@ on: torch-version: description: 'Torch version' required: true - default: '2.8.0' + default: '2.11.0' cuda-version: description: 'CUDA version' required: true - default: '12.9.1' + default: '13.0.0' cudnn-version: description: 'CUDNN version' required: true @@ -75,8 +75,8 @@ jobs: "os": ["ubuntu-22.04", "ubuntu-22.04-arm"], "release-version": ["${{ github.event.release.tag_name }}"], "python-version": ["3.12"], - "torch-version": ["2.8.0"], - "cuda-version": ["12.9.1"], + "torch-version": ["2.11.0"], + "cuda-version": ["13.0.0"], "cudnn-version": ["9"], "cxx11_abi": ["TRUE"] }' | jq -rc) @@ -166,6 +166,71 @@ jobs: asset_name: ${{ steps.build-pytorch-wheel.outputs.wheel_name }} asset_content_type: application/* + build_core_wheel: + name: Build Core Wheel (ubuntu-22.04-arm, CUDA 13) + runs-on: ubuntu-22.04-arm + needs: pre-flight + if: ${{ github.event_name == 'release' || inputs.ngc-image == '' }} + steps: + - name: 'Checkout' + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release-version }} + submodules: recursive + + - name: Move /var/lib/docker/ + run: sudo mv /var/lib/docker/ "${GITHUB_WORKSPACE}/docker" + + - name: Maximize build space + uses: easimon/maximize-build-space@c28619d8999a147d5e09c1199f84ff6af6ad5794 + with: + root-reserve-mb: 5120 + temp-reserve-mb: 32 + swap-size-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + build-mount-path: '/var/lib/docker/' + + - name: Restore /var/lib/docker/ + run: sudo sh -c "mv ${GITHUB_WORKSPACE}/docker/* /var/lib/docker" + + - name: Build CUDA 13 core wheel + id: build-core-wheel + run: | + docker build \ + --build-arg CUDA_MAJOR=13 \ + --build-arg CUDA_MINOR=0 \ + --build-arg BUILD_METAPACKAGE=false \ + --build-arg BUILD_COMMON=true \ + --build-arg BUILD_PYTORCH=false \ + --build-arg BUILD_JAX=false \ + --tag transformer-engine-core-wheel \ + --file build_tools/wheel_utils/Dockerfile.aarch \ + . + + mkdir -p dist + docker run --rm \ + --volume "$(pwd)/dist:/wheelhouse" \ + transformer-engine-core-wheel + + wheel_path=$(find dist -maxdepth 1 -type f \ + -name 'transformer_engine_cu13-*-manylinux_2_28_aarch64.whl' \ + -print -quit) + test -n "$wheel_path" + echo "wheel_name=$(basename "$wheel_path")" | tee -a "$GITHUB_OUTPUT" + + - name: Upload release asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.pre-flight.outputs.release-assets-url }} + asset_path: ./dist/${{ steps.build-core-wheel.outputs.wheel_name }} + asset_name: ${{ steps.build-core-wheel.outputs.wheel_name }} + asset_content_type: application/* + build_wheels_for_ngc: name: Build Wheels for NGC PyTorch images runs-on: ${{ matrix.os }} diff --git a/build_tools/wheel_utils/Dockerfile.aarch b/build_tools/wheel_utils/Dockerfile.aarch index c040dadcdb..d0b2ea74ed 100644 --- a/build_tools/wheel_utils/Dockerfile.aarch +++ b/build_tools/wheel_utils/Dockerfile.aarch @@ -25,13 +25,12 @@ ENV CUDA_MAJOR=${CUDA_MAJOR} RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/sbsa/cuda-rhel8.repo RUN dnf -y install cuda-compiler-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 \ cuda-libraries-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 \ - cuda-libraries-devel-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 + cuda-libraries-devel-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 \ + cuda-nvtx-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 RUN dnf -y install --allowerasing cudnn9-cuda-${CUDA_MAJOR} RUN dnf clean all RUN rm -rf /var/cache/dnf/* RUN echo "/usr/local/cuda/lib64" >> /etc/ld.so.conf.d/999_nvidia_cuda.conf -RUN dnf -y install cuda-toolkit-${CUDA_MAJOR} -RUN dnf clean all RUN dnf -y install glog.aarch64 glog-devel.aarch64 RUN dnf -y install libnccl libnccl-devel libnccl-static From c1716d6c02178f4f87a90a26f26c52e03a0bba8d Mon Sep 17 00:00:00 2001 From: bvolpato Date: Tue, 4 Aug 2026 00:49:12 -0400 Subject: [PATCH 2/3] [CI] Preserve existing release wheel defaults Signed-off-by: bvolpato --- .../workflows/attach-wheels-to-release.yml | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/attach-wheels-to-release.yml b/.github/workflows/attach-wheels-to-release.yml index 5568fa170f..bf520ac64f 100644 --- a/.github/workflows/attach-wheels-to-release.yml +++ b/.github/workflows/attach-wheels-to-release.yml @@ -32,11 +32,11 @@ on: torch-version: description: 'Torch version' required: true - default: '2.11.0' + default: '2.8.0' cuda-version: description: 'CUDA version' required: true - default: '13.0.0' + default: '12.9.1' cudnn-version: description: 'CUDNN version' required: true @@ -75,10 +75,30 @@ jobs: "os": ["ubuntu-22.04", "ubuntu-22.04-arm"], "release-version": ["${{ github.event.release.tag_name }}"], "python-version": ["3.12"], - "torch-version": ["2.11.0"], - "cuda-version": ["13.0.0"], + "torch-version": ["2.8.0"], + "cuda-version": ["12.9.1"], "cudnn-version": ["9"], - "cxx11_abi": ["TRUE"] + "cxx11_abi": ["TRUE"], + "include": [ + { + "os": "ubuntu-22.04", + "release-version": "${{ github.event.release.tag_name }}", + "python-version": "3.12", + "torch-version": "2.11.0", + "cuda-version": "13.0.0", + "cudnn-version": "9", + "cxx11_abi": "TRUE" + }, + { + "os": "ubuntu-22.04-arm", + "release-version": "${{ github.event.release.tag_name }}", + "python-version": "3.12", + "torch-version": "2.11.0", + "cuda-version": "13.0.0", + "cudnn-version": "9", + "cxx11_abi": "TRUE" + } + ] }' | jq -rc) else MATRIX=$(echo '{ @@ -173,7 +193,7 @@ jobs: if: ${{ github.event_name == 'release' || inputs.ngc-image == '' }} steps: - name: 'Checkout' - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release-version }} submodules: recursive @@ -222,7 +242,7 @@ jobs: echo "wheel_name=$(basename "$wheel_path")" | tee -a "$GITHUB_OUTPUT" - name: Upload release asset - uses: actions/upload-release-asset@v1 + uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 # v1.0.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: From 2da90bf8c1b8c034e874cc506bd4f170c32d3cc0 Mon Sep 17 00:00:00 2001 From: bvolpato Date: Sat, 15 Aug 2026 22:46:36 -0400 Subject: [PATCH 3/3] [CI] Validate CUDA 13 PyTorch release wheels Signed-off-by: bvolpato --- .../actions/build-pytorch-wheel/Dockerfile | 2 +- .../actions/build-pytorch-wheel/action.yml | 33 ++++++--- .github/actions/build-pytorch-wheel/build.sh | 67 +++++++++++++++-- .../workflows/attach-wheels-to-release.yml | 73 ++----------------- build_tools/wheel_utils/Dockerfile.aarch | 5 +- 5 files changed, 95 insertions(+), 85 deletions(-) diff --git a/.github/actions/build-pytorch-wheel/Dockerfile b/.github/actions/build-pytorch-wheel/Dockerfile index d3b0bd6f98..e526bae7ff 100644 --- a/.github/actions/build-pytorch-wheel/Dockerfile +++ b/.github/actions/build-pytorch-wheel/Dockerfile @@ -9,7 +9,7 @@ ENV DEBIAN_FRONTEND=noninteractive ENV CUDA_HOME=/usr/local/cuda ENV PATH=$PATH:$CUDA_HOME/bin ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH -ENV TORCH_CUDA_ARCH_LIST="6.0;6.1;7.0;7.5;8.0;8.6;9.0" +ENV TORCH_CUDA_ARCH_LIST="6.0;6.1;7.0;7.5;8.0;8.6;9.0;10.0" ARG PYTHON_VERSION=3.12 ARG TORCH_VERSION=2.9.1 diff --git a/.github/actions/build-pytorch-wheel/action.yml b/.github/actions/build-pytorch-wheel/action.yml index 526f121d3b..1d39da4551 100644 --- a/.github/actions/build-pytorch-wheel/action.yml +++ b/.github/actions/build-pytorch-wheel/action.yml @@ -30,6 +30,10 @@ inputs: aarch: description: 'The architecture to use for the build' required: true + smoke-test: + description: 'Install and import the wheel after building it' + required: false + default: 'false' outputs: wheel_name: description: 'The name of the built wheel' @@ -93,24 +97,35 @@ runs: shell: bash -euxo pipefail {0} id: build_wheel env: + AARCH: ${{ inputs.aarch }} CXX11_ABI: ${{ inputs.cxx11_abi }} + PYTHON_VERSION: ${{ inputs.python-version }} + RELEASE_VERSION: ${{ inputs.release-version }} + SMOKE_TEST: ${{ inputs.smoke-test }} run: | echo ::group::Build wheel - EXIT_CODE=$(docker run \ + set +e + docker run \ --rm \ --shm-size=64g \ --workdir /workspace/transformer_engine/pytorch \ - --volume $(pwd):/workspace \ - --volume $GITHUB_OUTPUT:$GITHUB_OUTPUT \ + --volume "$(pwd):/workspace" \ + --volume "$GITHUB_OUTPUT:$GITHUB_OUTPUT" \ + -e AARCH="$AARCH" \ -e PIP_CONSTRAINT= \ - -e CXX11_ABI=$CXX11_ABI \ - -e GITHUB_OUTPUT=$GITHUB_OUTPUT \ - transformer-engine-build bash /workspace/build-tools/.github/actions/build-pytorch-wheel/build.sh | tail -n 1) - - # Do not fail the job if timeout killed the build - exit $EXIT_CODE + -e CXX11_ABI="$CXX11_ABI" \ + -e GITHUB_OUTPUT="$GITHUB_OUTPUT" \ + -e MAX_JOBS \ + -e NVTE_FRAMEWORK \ + -e PYTHON_VERSION="$PYTHON_VERSION" \ + -e RELEASE_VERSION="$RELEASE_VERSION" \ + -e SMOKE_TEST="$SMOKE_TEST" \ + transformer-engine-build bash /workspace/build-tools/.github/actions/build-pytorch-wheel/build.sh + exit_code=$? + set -e echo ::endgroup:: + exit "$exit_code" - name: Log Built Wheels shell: bash -euxo pipefail {0} diff --git a/.github/actions/build-pytorch-wheel/build.sh b/.github/actions/build-pytorch-wheel/build.sh index 9a0920e2be..655f1938cd 100644 --- a/.github/actions/build-pytorch-wheel/build.sh +++ b/.github/actions/build-pytorch-wheel/build.sh @@ -14,13 +14,68 @@ export PIP_CONSTRAINT= pip install wheel packaging nvidia-mathdx ninja pybind11 # 5h timeout since GH allows max 6h and we want some buffer -EXIT_CODE=0 -timeout 5h python setup.py bdist_wheel --dist-dir=dist || EXIT_CODE=$? +timeout 5h python setup.py bdist_wheel --dist-dir=dist + +mapfile -t wheel_paths < <(find dist -maxdepth 1 -type f -name '*.whl' -print) +if [[ ${#wheel_paths[@]} -ne 1 ]]; then + echo "Expected one built wheel, found ${#wheel_paths[@]}" >&2 + exit 1 +fi +built_wheel=${wheel_paths[0]} + +if [[ -n "${AARCH:-}" ]]; then + case "$AARCH" in + x86_64) platform_tag=linux_x86_64 ;; + sbsa) platform_tag=linux_aarch64 ;; + *) echo "Unsupported wheel architecture: $AARCH" >&2; exit 1 ;; + esac + + python_tag="cp${PYTHON_VERSION//./}" + expected_tag="${python_tag}-${python_tag}-${platform_tag}" + python - "$built_wheel" "$expected_tag" <<'PY' +import sys +import zipfile + +wheel_path, expected_tag = sys.argv[1:] +with zipfile.ZipFile(wheel_path) as wheel: + wheel_metadata = next(name for name in wheel.namelist() if name.endswith(".dist-info/WHEEL")) + tags = { + line.removeprefix("Tag:").strip() + for line in wheel.read(wheel_metadata).decode().splitlines() + if line.startswith("Tag:") + } +if expected_tag not in tags: + raise SystemExit(f"Expected wheel tag {expected_tag}, found {sorted(tags)}") +PY -if [ $EXIT_CODE -eq 0 ]; then wheel_name=$(python -c "import setup; print(setup.get_wheel_url()[1])" | tail -n 1) - ls dist/*whl |xargs -I {} mv {} dist/${wheel_name} - echo "wheel_name=${wheel_name}" | tee -a "$GITHUB_OUTPUT" + expected_suffix="cxx11abi${CXX11_ABI}-${expected_tag}.whl" + if [[ "$wheel_name" != *"$expected_suffix" ]]; then + echo "Expected wheel filename suffix $expected_suffix, found $wheel_name" >&2 + exit 1 + fi +else + wheel_name=$(python -c "import setup; print(setup.get_wheel_url()[1])" | tail -n 1) +fi + +if [[ "${SMOKE_TEST:-false}" == "true" ]]; then + release_version=${RELEASE_VERSION:-} + release_version=${release_version#v} + if [[ -z "$release_version" ]]; then + echo "Release version is required for the install smoke test" >&2 + exit 1 + fi + pip install --no-cache-dir "transformer-engine==${release_version}" "$built_wheel" + + cuda_stub_dir=$(mktemp -d) + test -f /usr/local/cuda/lib64/stubs/libcuda.so + ln -s /usr/local/cuda/lib64/stubs/libcuda.so "$cuda_stub_dir/libcuda.so.1" + ( + cd /tmp + LD_LIBRARY_PATH="$cuda_stub_dir:${LD_LIBRARY_PATH:-}" python -c \ + "import transformer_engine.pytorch; print('Transformer Engine import OK')" + ) fi -echo $EXIT_CODE +mv "$built_wheel" "dist/$wheel_name" +echo "wheel_name=${wheel_name}" | tee -a "$GITHUB_OUTPUT" diff --git a/.github/workflows/attach-wheels-to-release.yml b/.github/workflows/attach-wheels-to-release.yml index bf520ac64f..d00e129c59 100644 --- a/.github/workflows/attach-wheels-to-release.yml +++ b/.github/workflows/attach-wheels-to-release.yml @@ -79,6 +79,7 @@ jobs: "cuda-version": ["12.9.1"], "cudnn-version": ["9"], "cxx11_abi": ["TRUE"], + "smoke-test": ["false"], "include": [ { "os": "ubuntu-22.04", @@ -87,7 +88,8 @@ jobs: "torch-version": "2.11.0", "cuda-version": "13.0.0", "cudnn-version": "9", - "cxx11_abi": "TRUE" + "cxx11_abi": "TRUE", + "smoke-test": "true" }, { "os": "ubuntu-22.04-arm", @@ -96,7 +98,8 @@ jobs: "torch-version": "2.11.0", "cuda-version": "13.0.0", "cudnn-version": "9", - "cxx11_abi": "TRUE" + "cxx11_abi": "TRUE", + "smoke-test": "true" } ] }' | jq -rc) @@ -171,6 +174,7 @@ jobs: torch-version: ${{ matrix.torch-version }} cxx11_abi: ${{ matrix.cxx11_abi }} aarch: ${{ matrix.os == 'ubuntu-22.04' && 'x86_64' || 'sbsa' }} + smoke-test: ${{ matrix.smoke-test || 'false' }} env: NVTE_FRAMEWORK: pytorch MAX_JOBS: 1 @@ -186,71 +190,6 @@ jobs: asset_name: ${{ steps.build-pytorch-wheel.outputs.wheel_name }} asset_content_type: application/* - build_core_wheel: - name: Build Core Wheel (ubuntu-22.04-arm, CUDA 13) - runs-on: ubuntu-22.04-arm - needs: pre-flight - if: ${{ github.event_name == 'release' || inputs.ngc-image == '' }} - steps: - - name: 'Checkout' - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - with: - ref: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release-version }} - submodules: recursive - - - name: Move /var/lib/docker/ - run: sudo mv /var/lib/docker/ "${GITHUB_WORKSPACE}/docker" - - - name: Maximize build space - uses: easimon/maximize-build-space@c28619d8999a147d5e09c1199f84ff6af6ad5794 - with: - root-reserve-mb: 5120 - temp-reserve-mb: 32 - swap-size-mb: 10240 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - build-mount-path: '/var/lib/docker/' - - - name: Restore /var/lib/docker/ - run: sudo sh -c "mv ${GITHUB_WORKSPACE}/docker/* /var/lib/docker" - - - name: Build CUDA 13 core wheel - id: build-core-wheel - run: | - docker build \ - --build-arg CUDA_MAJOR=13 \ - --build-arg CUDA_MINOR=0 \ - --build-arg BUILD_METAPACKAGE=false \ - --build-arg BUILD_COMMON=true \ - --build-arg BUILD_PYTORCH=false \ - --build-arg BUILD_JAX=false \ - --tag transformer-engine-core-wheel \ - --file build_tools/wheel_utils/Dockerfile.aarch \ - . - - mkdir -p dist - docker run --rm \ - --volume "$(pwd)/dist:/wheelhouse" \ - transformer-engine-core-wheel - - wheel_path=$(find dist -maxdepth 1 -type f \ - -name 'transformer_engine_cu13-*-manylinux_2_28_aarch64.whl' \ - -print -quit) - test -n "$wheel_path" - echo "wheel_name=$(basename "$wheel_path")" | tee -a "$GITHUB_OUTPUT" - - - name: Upload release asset - uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 # v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.pre-flight.outputs.release-assets-url }} - asset_path: ./dist/${{ steps.build-core-wheel.outputs.wheel_name }} - asset_name: ${{ steps.build-core-wheel.outputs.wheel_name }} - asset_content_type: application/* - build_wheels_for_ngc: name: Build Wheels for NGC PyTorch images runs-on: ${{ matrix.os }} diff --git a/build_tools/wheel_utils/Dockerfile.aarch b/build_tools/wheel_utils/Dockerfile.aarch index d0b2ea74ed..c040dadcdb 100644 --- a/build_tools/wheel_utils/Dockerfile.aarch +++ b/build_tools/wheel_utils/Dockerfile.aarch @@ -25,12 +25,13 @@ ENV CUDA_MAJOR=${CUDA_MAJOR} RUN dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/sbsa/cuda-rhel8.repo RUN dnf -y install cuda-compiler-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 \ cuda-libraries-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 \ - cuda-libraries-devel-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 \ - cuda-nvtx-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 + cuda-libraries-devel-${CUDA_MAJOR}-${CUDA_MINOR}.aarch64 RUN dnf -y install --allowerasing cudnn9-cuda-${CUDA_MAJOR} RUN dnf clean all RUN rm -rf /var/cache/dnf/* RUN echo "/usr/local/cuda/lib64" >> /etc/ld.so.conf.d/999_nvidia_cuda.conf +RUN dnf -y install cuda-toolkit-${CUDA_MAJOR} +RUN dnf clean all RUN dnf -y install glog.aarch64 glog-devel.aarch64 RUN dnf -y install libnccl libnccl-devel libnccl-static