From c82cbe1eb10542f99984a08b14e6b2aac685010d Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 23 Jul 2026 14:37:28 -0700 Subject: [PATCH] [SPARK-58302][INFRA] Remove the unused dev/infra/Dockerfile base image Co-authored-by: Isaac --- .github/workflows/build_and_test.yml | 9 - .../workflows/build_infra_images_cache.yml | 14 -- dev/infra/Dockerfile | 155 ------------------ 3 files changed, 178 deletions(-) delete mode 100644 dev/infra/Dockerfile diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b41515df16176..8e2ea2884d1ee 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -70,7 +70,6 @@ jobs: required: ${{ steps.set-outputs.outputs.required }} # Pinned so every downstream job checks out the same snapshot, even if `master` advances mid-run. head_sha: ${{ steps.resolve-sha.outputs.head_sha }} - image_url: ${{ steps.infra-image-outputs.outputs.image_url }} image_docs_url: ${{ steps.infra-image-docs-outputs.outputs.image_docs_url }} image_docs_url_link: ${{ steps.infra-image-link.outputs.image_docs_url_link }} image_lint_url: ${{ steps.infra-image-lint-outputs.outputs.image_lint_url }} @@ -204,14 +203,6 @@ jobs: exit 1 fi fi - - name: Generate infra image URL - id: infra-image-outputs - run: | - # Convert to lowercase to meet Docker repo name requirement - REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') - IMG_NAME="apache-spark-ci-image:${{ inputs.branch }}-${{ github.run_id }}" - IMG_URL="ghcr.io/$REPO_OWNER/$IMG_NAME" - echo "image_url=$IMG_URL" >> $GITHUB_OUTPUT - name: Generate infra image URL (Documentation) id: infra-image-docs-outputs run: | diff --git a/.github/workflows/build_infra_images_cache.yml b/.github/workflows/build_infra_images_cache.yml index e20a7d1c3ef80..9800107fc369c 100644 --- a/.github/workflows/build_infra_images_cache.yml +++ b/.github/workflows/build_infra_images_cache.yml @@ -33,7 +33,6 @@ on: - 'master' - 'branch-*' paths: - - 'dev/infra/Dockerfile' - 'dev/spark-test-image/docs/Dockerfile' - 'dev/spark-test-image/lint/Dockerfile' - 'dev/spark-test-image/sparkr/Dockerfile' @@ -69,18 +68,6 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push - id: docker_build - continue-on-error: true - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f - with: - context: ./dev/infra/ - push: true - tags: ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{ github.ref_name }}-static - cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{ github.ref_name }} - cache-to: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{ github.ref_name }},mode=max - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - name: Build and push (Documentation) if: hashFiles('dev/spark-test-image/docs/Dockerfile') != '' id: docker_build_docs @@ -255,7 +242,6 @@ jobs: status=1 fi } - check "base" "${{ steps.docker_build.outcome }}" check "docs" "${{ steps.docker_build_docs.outcome }}" check "lint" "${{ steps.docker_build_lint.outcome }}" check "sparkr" "${{ steps.docker_build_sparkr.outcome }}" diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile deleted file mode 100644 index a54e270de5678..0000000000000 --- a/dev/infra/Dockerfile +++ /dev/null @@ -1,155 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Image for building and testing Spark branches. Based on Ubuntu 22.04. -# See also in https://hub.docker.com/_/ubuntu -FROM ubuntu:jammy-20240911.1 -LABEL org.opencontainers.image.authors="Apache Spark project " -LABEL org.opencontainers.image.licenses="Apache-2.0" -LABEL org.opencontainers.image.ref.name="Apache Spark Infra Image" -# Overwrite this label to avoid exposing the underlying Ubuntu OS version label -LABEL org.opencontainers.image.version="" - -ENV FULL_REFRESH_DATE=20260317 - -ENV DEBIAN_FRONTEND=noninteractive -ENV DEBCONF_NONINTERACTIVE_SEEN=true - -RUN echo 'deb https://mirrors.edge.kernel.org/ubuntu jammy main restricted universe multiverse' > /etc/apt/sources.list.d/mirror.list && \ - echo 'deb https://mirrors.edge.kernel.org/ubuntu jammy-updates main restricted universe multiverse' >> /etc/apt/sources.list.d/mirror.list && \ - echo 'deb https://mirrors.edge.kernel.org/ubuntu jammy-security main restricted universe multiverse' >> /etc/apt/sources.list.d/mirror.list - -RUN apt-get update && apt-get install -y \ - build-essential \ - ca-certificates \ - curl \ - gfortran \ - git \ - gnupg \ - libcurl4-openssl-dev \ - libfontconfig1-dev \ - libfreetype6-dev \ - libfribidi-dev \ - libgit2-dev \ - libharfbuzz-dev \ - libjpeg-dev \ - liblapack-dev \ - libopenblas-dev \ - libpng-dev \ - libpython3-dev \ - libssl-dev \ - libtiff5-dev \ - libuv1-dev \ - libwebp-dev \ - libxml2-dev \ - nodejs \ - npm \ - openjdk-17-jdk-headless \ - pandoc \ - pkg-config \ - python3.10 \ - python3-psutil \ - qpdf \ - r-base \ - ruby \ - ruby-dev \ - software-properties-common \ - wget \ - zlib1g-dev \ - zstd \ - && rm -rf /var/lib/apt/lists/* - - -RUN echo 'deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/' >> /etc/apt/sources.list -RUN gpg --keyserver hkps://keyserver.ubuntu.com --recv-key E298A3A825C0D65DFD57CBB651716619E084DAB9 -RUN gpg -a --export E084DAB9 > /etc/apt/trusted.gpg.d/cran.asc -RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/' - -# See more in SPARK-39959, roxygen2 < 7.2.1 -RUN Rscript -e "install.packages(c('remotes', 'knitr', 'markdown', \ - 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', \ - 'ggplot2', 'mvtnorm', 'statmod', 'xml2'), repos='https://cloud.r-project.org/')" && \ - Rscript -e "remotes::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ - Rscript -e "remotes::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" && \ - Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" && \ - Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" - -# See more in SPARK-39735 -ENV R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library" - -ARG BASIC_PIP_PKGS="numpy pyarrow>=18.0.0 six==1.16.0 pandas==2.3.3 scipy plotly>=4.8 mlflow>=2.8.1 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2" -# Python deps for Spark Connect -ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5 googleapis-common-protos==1.71.0 graphviz==0.20.3" - -# Install Python 3.10 packages -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 -RUN python3.10 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this -RUN python3.10 -m pip install --ignore-installed 'six==1.16.0' # Avoid `python3-six` installation -RUN python3.10 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \ - python3.10 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - python3.10 -m pip install deepspeed torcheval && \ - python3.10 -m pip cache purge - -# Install Python 3.9 -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update && apt-get install -y \ - python3.9 python3.9-distutils \ - && rm -rf /var/lib/apt/lists/* -RUN curl -sS https://bootstrap.pypa.io/pip/3.9/get-pip.py | python3.9 -RUN python3.9 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this -RUN python3.9 -m pip install --force $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \ - python3.9 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - python3.9 -m pip install torcheval && \ - python3.9 -m pip cache purge - -# Install Python 3.11 at the last stage to avoid breaking the existing Python installations -RUN apt-get update && apt-get install -y \ - python3.11 \ - && rm -rf /var/lib/apt/lists/* -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 -RUN python3.11 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this -RUN python3.11 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS && \ - python3.11 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - python3.11 -m pip install deepspeed torcheval && \ - python3.11 -m pip cache purge - -# Install Python 3.12 at the last stage to avoid breaking the existing Python installations -RUN apt-get update && apt-get install -y \ - python3.12 \ - && rm -rf /var/lib/apt/lists/* -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 -RUN python3.12 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this -RUN python3.12 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS lxml && \ - python3.12 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - python3.12 -m pip install torcheval && \ - python3.12 -m pip cache purge - -# Install Python 3.13 at the last stage to avoid breaking the existing Python installations -RUN apt-get update && apt-get install -y \ - python3.13 \ - && rm -rf /var/lib/apt/lists/* -RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 -RUN python3.13 -m pip install --ignore-installed 'blinker>=1.6.2' # mlflow needs this -RUN python3.13 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS lxml && \ - python3.13 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \ - python3.13 -m pip install torcheval && \ - python3.13 -m pip cache purge - -# Remove unused installation packages to free up disk space -RUN apt-get remove --purge -y 'humanity-icon-theme' 'nodejs-doc' -RUN apt-get autoremove --purge -y -RUN apt-get clean