apache · move-hoon · Aug 11, 2026
diff --git a/docs/setup/deployment/spark_cluster_mode.md b/docs/setup/deployment/spark_cluster_mode.md
@@ -37,6 +37,8 @@ You can simply set up Spark standalone environment with below steps.
 
 ### 1. Build Docker file
 You can find docker script files under `scripts/docker/spark-cluster-managers`.
+The image uses the official Apache Spark Ubuntu image with Java 11. Its Spark 3.5.8 and Scala 2.12 defaults match Zeppelin's build versions.
+You can override these versions with the `JAVA_VERSION`, `SPARK_VERSION`, and `SCALA_VERSION` build arguments when a matching Apache Spark image tag is available.
 
 ```bash
 cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_standalone
@@ -82,6 +84,10 @@ You can simply set up [Spark on YARN](http://spark.apache.org/docs/latest/runnin
 
 ### 1. Build Docker file
 You can find docker script files under `scripts/docker/spark-cluster-managers`.
+The image uses the official Apache Spark Ubuntu image with Java 11. Its Spark 3.5.8, Scala 2.12, and Hadoop 3.3.6 defaults match Zeppelin's build versions.
+You can override `JAVA_VERSION`, `SPARK_VERSION`, and `SCALA_VERSION` when a matching Apache Spark image tag is available.
+The image supports `linux/amd64` and `linux/arm64`; override `HADOOP_VERSION` only when the Apache Hadoop archive provides a release artifact for the selected target architecture.
+Hadoop 3.3.x supports Java 8 and Java 11 at runtime, so keep `JAVA_VERSION=11` when using the default Hadoop 3.3.6. Use another Java version only with a Hadoop release that officially supports it.
 
 ```bash
 cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_yarn_cluster
@@ -108,7 +114,7 @@ docker run -it \
  -p 50010:50010 \
  -p 50075:50075 \
  -p 50020:50020 \
- -p 50070:50070 \
+ -p 50070:9870 \
  --name spark_yarn \
  -h sparkmaster \
  spark_yarn bash;
@@ -136,9 +142,7 @@ export SPARK_HOME=[your_spark_home_path]
 
 `HADOOP_CONF_DIR`(Hadoop configuration path) is defined in `/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf`.
 
-Don't forget to set Spark `spark.master` as `yarn-client` in Zeppelin **Interpreters** setting page like below.
-
-<img src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png" />
+Don't forget to set Spark `spark.master` as `yarn` and `spark.submit.deployMode` as `client` in Zeppelin **Interpreters** setting page.
 
 ### 5. Run Zeppelin with Spark interpreter
 After running a single paragraph with Spark interpreter in Zeppelin, browse `http://<hostname>:8088/cluster/apps` and check Zeppelin application is running well or not.

diff --git a/scripts/docker/spark-cluster-managers/spark_standalone/Dockerfile b/scripts/docker/spark-cluster-managers/spark_standalone/Dockerfile
@@ -12,39 +12,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM centos:centos7
+ARG SPARK_VERSION=3.5.8
+ARG SCALA_VERSION=2.12
+ARG JAVA_VERSION=11
+FROM apache/spark:${SPARK_VERSION}-scala${SCALA_VERSION}-java${JAVA_VERSION}-python3-ubuntu
 
-ENV SPARK_PROFILE 2.4
-ENV SPARK_VERSION 2.4.8
-ENV HADOOP_PROFILE 2.7
-ENV SPARK_HOME /usr/local/spark
+USER root
 
-# Update the image with the latest packages
-RUN yum update -y; yum clean all
-
-# Get utils
-RUN yum install -y \
-wget \
-tar \
-curl \
-&& \
-yum clean all
-
-# Remove old jdk
-RUN yum remove java; yum remove jdk
-
-# install jdk7
-RUN yum install -y java-1.7.0-openjdk-devel
-ENV JAVA_HOME /usr/lib/jvm/java
-ENV PATH $PATH:$JAVA_HOME/bin
-
-# install spark
-RUN curl -s https://downloads.apache.org/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz | tar -xz -C /usr/local/
-RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      ca-certificates \
+      openssh-server \
+      procps \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /run/sshd
 
 # update boot script
 COPY entrypoint.sh /etc/entrypoint.sh
-RUN chown root.root /etc/entrypoint.sh
+RUN chown root:root /etc/entrypoint.sh
 RUN chmod 700 /etc/entrypoint.sh
 
 #spark

diff --git a/scripts/docker/spark-cluster-managers/spark_standalone/entrypoint.sh b/scripts/docker/spark-cluster-managers/spark_standalone/entrypoint.sh
@@ -17,14 +17,14 @@
 export SPARK_MASTER_PORT=7077
 
 # run spark 
-cd /usr/local/spark/sbin
+cd $SPARK_HOME/sbin
 ./start-master.sh
-./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT
+./start-worker.sh spark://`hostname`:$SPARK_MASTER_PORT
 
 CMD=${1:-"exit 0"}
 if [[ "$CMD" == "-d" ]];
 then
-	service sshd stop
+	service ssh stop
 	/usr/sbin/sshd -D -d
 else
 	/bin/bash -c "$*"

diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/Dockerfile b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/Dockerfile
@@ -12,91 +12,84 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-FROM centos:centos7
-
-ENV SPARK_PROFILE 2.4
-ENV SPARK_VERSION 2.4.0
-ENV HADOOP_PROFILE 2.7
-ENV HADOOP_VERSION 2.7.0
-
-# Update the image with the latest packages
-RUN yum update -y; yum clean all
-
-# Get utils
-RUN yum install -y \
-wget \
-tar \
-curl \
-&& \
-yum clean all
-
-# Remove old jdk
-RUN yum remove java; yum remove jdk
-
-# install jdk7
-RUN yum install -y java-1.7.0-openjdk-devel
-ENV JAVA_HOME /usr/lib/jvm/java
-ENV PATH $PATH:$JAVA_HOME/bin
-
-# install hadoop 
-RUN yum install -y curl which tar sudo openssh-server openssh-clients rsync
+ARG SPARK_VERSION=3.5.8
+ARG SCALA_VERSION=2.12
+ARG JAVA_VERSION=11
+FROM apache/spark:${SPARK_VERSION}-scala${SCALA_VERSION}-java${JAVA_VERSION}-python3-ubuntu
+
+USER root
+
+ARG HADOOP_VERSION=3.3.6
+ARG TARGETARCH
+ENV HADOOP_VERSION=${HADOOP_VERSION} \
+    HADOOP_PREFIX=/usr/local/hadoop \
+    HADOOP_COMMON_HOME=/usr/local/hadoop \
+    HADOOP_HDFS_HOME=/usr/local/hadoop \
+    HADOOP_MAPRED_HOME=/usr/local/hadoop \
+    HADOOP_YARN_HOME=/usr/local/hadoop \
+    HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop \
+    YARN_CONF_DIR=/usr/local/hadoop/etc/hadoop \
+    PATH=${PATH}:/usr/local/hadoop/bin
+
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      ca-certificates \
+      curl \
+      openssh-server \
+      procps \
+      rsync \
+    && rm -rf /var/lib/apt/lists/* \
+    && mkdir -p /run/sshd /root/.ssh
 
 # hadoop
-RUN curl -s "https://www.apache.org/dyn/closer.lua/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz?action=download" | tar -xz -C /usr/local/
-RUN cd /usr/local && ln -s ./hadoop-$HADOOP_VERSION hadoop
-
-ENV HADOOP_PREFIX /usr/local/hadoop
-ENV HADOOP_COMMON_HOME /usr/local/hadoop
-ENV HADOOP_HDFS_HOME /usr/local/hadoop
-ENV HADOOP_MAPRED_HOME /usr/local/hadoop
-ENV HADOOP_YARN_HOME /usr/local/hadoop
-ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop
-
-RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/lib/jvm/jre-1.7.0-openjdk\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
-RUN sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+RUN case "$TARGETARCH" in \
+      amd64) HADOOP_ARCHIVE="hadoop-$HADOOP_VERSION.tar.gz" ;; \
+      arm64) HADOOP_ARCHIVE="hadoop-$HADOOP_VERSION-aarch64.tar.gz" ;; \
+      *) echo "Unsupported architecture: $TARGETARCH" >&2; exit 1 ;; \
+    esac \
+    && HADOOP_URL="https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/$HADOOP_ARCHIVE" \
+    && curl -fL "$HADOOP_URL" -o "/tmp/$HADOOP_ARCHIVE" \
+    && curl -fL "$HADOOP_URL.sha512" -o "/tmp/$HADOOP_ARCHIVE.sha512" \
+    && HADOOP_SHA512="$(sed -n 's/^SHA512 ([^)]*) = \([[:xdigit:]]\{128\}\)$/\1/p' "/tmp/$HADOOP_ARCHIVE.sha512")" \
+    && test -n "$HADOOP_SHA512" \
+    && echo "$HADOOP_SHA512  /tmp/$HADOOP_ARCHIVE" | sha512sum -c - \
+    && tar -xzf "/tmp/$HADOOP_ARCHIVE" -C /usr/local/ \
+    && rm "/tmp/$HADOOP_ARCHIVE" "/tmp/$HADOOP_ARCHIVE.sha512" \
+    && ln -s /usr/local/hadoop-$HADOOP_VERSION $HADOOP_PREFIX
+
+RUN printf '\nexport JAVA_HOME=%s\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\nexport HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop\nexport HDFS_NAMENODE_USER=root\nexport HDFS_DATANODE_USER=root\nexport HDFS_SECONDARYNAMENODE_USER=root\nexport YARN_RESOURCEMANAGER_USER=root\nexport YARN_NODEMANAGER_USER=root\n' "$JAVA_HOME" \
+    >> $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
 
 RUN mkdir $HADOOP_PREFIX/input
 RUN cp $HADOOP_PREFIX/etc/hadoop/*.xml $HADOOP_PREFIX/input
 
 # hadoop configurations
-ADD hdfs_conf/core-site.xml $HADOOP_PREFIX/etc/hadoop/core-site.xml
-ADD hdfs_conf/hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
-ADD hdfs_conf/mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
-ADD hdfs_conf/yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
+COPY hdfs_conf/core-site.xml $HADOOP_PREFIX/etc/hadoop/core-site.xml
+COPY hdfs_conf/hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml
+COPY hdfs_conf/mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml
+COPY hdfs_conf/yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml
 
 RUN mkdir /data/
 RUN chmod 777 /data/
 RUN $HADOOP_PREFIX/bin/hdfs namenode -format
 
-RUN rm  /usr/local/hadoop/lib/native/*
-RUN curl -Ls http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-$HADOOP_VERSION.tar|tar -x -C /usr/local/hadoop/lib/native/
-
-# install spark
-RUN curl -s "https://www.apache.org/dyn/closer.lua/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz?action=download" | tar -xz -C /usr/local/
-RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark
-ENV SPARK_HOME /usr/local/spark
-
-ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
-ENV PATH $PATH:$SPARK_HOME/bin:$HADOOP_PREFIX/bin
-
 # passwordless ssh
-RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
-RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key
+RUN ssh-keygen -A
 RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa
 RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
 
-ADD ssh_config /root/.ssh/config
+COPY ssh_config /root/.ssh/config
 RUN chmod 600 /root/.ssh/config
 RUN chown root:root /root/.ssh/config
 RUN chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh
 
 # update boot script
 COPY entrypoint.sh /etc/entrypoint.sh
-RUN chown root.root /etc/entrypoint.sh
+RUN chown root:root /etc/entrypoint.sh
 RUN chmod 700 /etc/entrypoint.sh
 
 # Hdfs ports
-EXPOSE 50010 50020 50070 50075 50090
+EXPOSE 50010 50020 50075 9868 9870
 # Mapred ports
 EXPOSE 9000 9001
 #Yarn ports

diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh
@@ -14,25 +14,37 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-echo 'hadoop' |passwd root --stdin
+set -euo pipefail
 
-: ${HADOOP_PREFIX:=/usr/local/hadoop}
+: "${HADOOP_PREFIX:=/usr/local/hadoop}"
 
-$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+. "$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh"
 
-rm /tmp/*.pid
+rm -f /tmp/*.pid
 
 # installing libraries if any - (resource urls added comma separated to the ACP system variable)
-cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do  echo == $cp; curl -LO $cp ; done; cd -
+cd "$HADOOP_PREFIX/share/hadoop/common"
+ACP_URLS="${ACP:-}"
+for cp in ${ACP_URLS//,/ }; do
+  echo "== $cp"
+  curl -fLO -- "$cp"
+done
+cd - > /dev/null
 
-cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties
+cp "$SPARK_HOME/conf/metrics.properties.template" "$SPARK_HOME/conf/metrics.properties" || true
 
 # start hadoop
-service sshd start
-$HADOOP_PREFIX/sbin/start-dfs.sh
-$HADOOP_PREFIX/sbin/start-yarn.sh
+service ssh start
+"$HADOOP_PREFIX/sbin/start-dfs.sh"
+"$HADOOP_PREFIX/sbin/start-yarn.sh"
 
-$HADOOP_PREFIX/bin/hdfs dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE/lib /spark
+"$HADOOP_PREFIX/bin/hdfs" dfsadmin -safemode leave \
+  && "$HADOOP_PREFIX/bin/hdfs" dfs -mkdir -p /spark
+if ! "$HADOOP_PREFIX/bin/hdfs" dfs -test -e /spark/.jars-upload-complete; then
+  "$HADOOP_PREFIX/bin/hdfs" dfs -rm -r -f /spark/jars
+  "$HADOOP_PREFIX/bin/hdfs" dfs -put "$SPARK_HOME/jars" /spark
+  "$HADOOP_PREFIX/bin/hdfs" dfs -touchz /spark/.jars-upload-complete
+fi
 
 # start spark
 export SPARK_MASTER_OPTS="-Dspark.driver.port=7001 -Dspark.fileserver.port=7002
@@ -46,14 +58,14 @@ export SPARK_WORKER_OPTS="-Dspark.driver.port=7001 -Dspark.fileserver.port=7002
 
 export SPARK_MASTER_PORT=7077
 
-cd /usr/local/spark/sbin
+cd "$SPARK_HOME/sbin"
 ./start-master.sh
-./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT
+./start-worker.sh "spark://$(hostname):$SPARK_MASTER_PORT"
 
 CMD=${1:-"exit 0"}
 if [[ "$CMD" == "-d" ]];
 then
-	service sshd stop
+	service ssh stop
 	/usr/sbin/sshd -D -d
 else
 	/bin/bash -c "$*"

diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/core-site.xml b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/core-site.xml
@@ -17,6 +17,6 @@ limitations under the License.
 <configuration>
   <property>
     <name>fs.defaultFS</name>
-    <value>hdfs://0.0.0.0:9000</value>
+    <value>hdfs://sparkmaster:9000</value>
   </property>
 </configuration>