diff --git a/docs/setup/deployment/spark_cluster_mode.md b/docs/setup/deployment/spark_cluster_mode.md index f6d4de8c71f..0b635fda487 100644 --- a/docs/setup/deployment/spark_cluster_mode.md +++ b/docs/setup/deployment/spark_cluster_mode.md @@ -37,6 +37,8 @@ You can simply set up Spark standalone environment with below steps. ### 1. Build Docker file You can find docker script files under `scripts/docker/spark-cluster-managers`. +The image uses the official Apache Spark Ubuntu image with Java 11. Its Spark 3.5.8 and Scala 2.12 defaults match Zeppelin's build versions. +You can override these versions with the `JAVA_VERSION`, `SPARK_VERSION`, and `SCALA_VERSION` build arguments when a matching Apache Spark image tag is available. ```bash cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_standalone @@ -82,6 +84,10 @@ You can simply set up [Spark on YARN](http://spark.apache.org/docs/latest/runnin ### 1. Build Docker file You can find docker script files under `scripts/docker/spark-cluster-managers`. +The image uses the official Apache Spark Ubuntu image with Java 11. Its Spark 3.5.8, Scala 2.12, and Hadoop 3.3.6 defaults match Zeppelin's build versions. +You can override `JAVA_VERSION`, `SPARK_VERSION`, and `SCALA_VERSION` when a matching Apache Spark image tag is available. +The image supports `linux/amd64` and `linux/arm64`; override `HADOOP_VERSION` only when the Apache Hadoop archive provides a release artifact for the selected target architecture. +Hadoop 3.3.x supports Java 8 and Java 11 at runtime, so keep `JAVA_VERSION=11` when using the default Hadoop 3.3.6. Use another Java version only with a Hadoop release that officially supports it. ```bash cd $ZEPPELIN_HOME/scripts/docker/spark-cluster-managers/spark_yarn_cluster @@ -108,7 +114,7 @@ docker run -it \ -p 50010:50010 \ -p 50075:50075 \ -p 50020:50020 \ - -p 50070:50070 \ + -p 50070:9870 \ --name spark_yarn \ -h sparkmaster \ spark_yarn bash; @@ -136,9 +142,7 @@ export SPARK_HOME=[your_spark_home_path] `HADOOP_CONF_DIR`(Hadoop configuration path) is defined in `/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf`. -Don't forget to set Spark `spark.master` as `yarn-client` in Zeppelin **Interpreters** setting page like below. - - +Don't forget to set Spark `spark.master` as `yarn` and `spark.submit.deployMode` as `client` in Zeppelin **Interpreters** setting page. ### 5. Run Zeppelin with Spark interpreter After running a single paragraph with Spark interpreter in Zeppelin, browse `http://:8088/cluster/apps` and check Zeppelin application is running well or not. diff --git a/scripts/docker/spark-cluster-managers/spark_standalone/Dockerfile b/scripts/docker/spark-cluster-managers/spark_standalone/Dockerfile index 6b3646e2a5a..ef4ad8b2a33 100644 --- a/scripts/docker/spark-cluster-managers/spark_standalone/Dockerfile +++ b/scripts/docker/spark-cluster-managers/spark_standalone/Dockerfile @@ -12,39 +12,24 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM centos:centos7 +ARG SPARK_VERSION=3.5.8 +ARG SCALA_VERSION=2.12 +ARG JAVA_VERSION=11 +FROM apache/spark:${SPARK_VERSION}-scala${SCALA_VERSION}-java${JAVA_VERSION}-python3-ubuntu -ENV SPARK_PROFILE 2.4 -ENV SPARK_VERSION 2.4.8 -ENV HADOOP_PROFILE 2.7 -ENV SPARK_HOME /usr/local/spark +USER root -# Update the image with the latest packages -RUN yum update -y; yum clean all - -# Get utils -RUN yum install -y \ -wget \ -tar \ -curl \ -&& \ -yum clean all - -# Remove old jdk -RUN yum remove java; yum remove jdk - -# install jdk7 -RUN yum install -y java-1.7.0-openjdk-devel -ENV JAVA_HOME /usr/lib/jvm/java -ENV PATH $PATH:$JAVA_HOME/bin - -# install spark -RUN curl -s https://downloads.apache.org/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz | tar -xz -C /usr/local/ -RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + openssh-server \ + procps \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /run/sshd # update boot script COPY entrypoint.sh /etc/entrypoint.sh -RUN chown root.root /etc/entrypoint.sh +RUN chown root:root /etc/entrypoint.sh RUN chmod 700 /etc/entrypoint.sh #spark diff --git a/scripts/docker/spark-cluster-managers/spark_standalone/entrypoint.sh b/scripts/docker/spark-cluster-managers/spark_standalone/entrypoint.sh index f4fded0dc9e..6e571114614 100755 --- a/scripts/docker/spark-cluster-managers/spark_standalone/entrypoint.sh +++ b/scripts/docker/spark-cluster-managers/spark_standalone/entrypoint.sh @@ -17,14 +17,14 @@ export SPARK_MASTER_PORT=7077 # run spark -cd /usr/local/spark/sbin +cd $SPARK_HOME/sbin ./start-master.sh -./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT +./start-worker.sh spark://`hostname`:$SPARK_MASTER_PORT CMD=${1:-"exit 0"} if [[ "$CMD" == "-d" ]]; then - service sshd stop + service ssh stop /usr/sbin/sshd -D -d else /bin/bash -c "$*" diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/Dockerfile b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/Dockerfile index 01b15308fdc..a945f3999af 100644 --- a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/Dockerfile +++ b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/Dockerfile @@ -12,91 +12,84 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM centos:centos7 - -ENV SPARK_PROFILE 2.4 -ENV SPARK_VERSION 2.4.0 -ENV HADOOP_PROFILE 2.7 -ENV HADOOP_VERSION 2.7.0 - -# Update the image with the latest packages -RUN yum update -y; yum clean all - -# Get utils -RUN yum install -y \ -wget \ -tar \ -curl \ -&& \ -yum clean all - -# Remove old jdk -RUN yum remove java; yum remove jdk - -# install jdk7 -RUN yum install -y java-1.7.0-openjdk-devel -ENV JAVA_HOME /usr/lib/jvm/java -ENV PATH $PATH:$JAVA_HOME/bin - -# install hadoop -RUN yum install -y curl which tar sudo openssh-server openssh-clients rsync +ARG SPARK_VERSION=3.5.8 +ARG SCALA_VERSION=2.12 +ARG JAVA_VERSION=11 +FROM apache/spark:${SPARK_VERSION}-scala${SCALA_VERSION}-java${JAVA_VERSION}-python3-ubuntu + +USER root + +ARG HADOOP_VERSION=3.3.6 +ARG TARGETARCH +ENV HADOOP_VERSION=${HADOOP_VERSION} \ + HADOOP_PREFIX=/usr/local/hadoop \ + HADOOP_COMMON_HOME=/usr/local/hadoop \ + HADOOP_HDFS_HOME=/usr/local/hadoop \ + HADOOP_MAPRED_HOME=/usr/local/hadoop \ + HADOOP_YARN_HOME=/usr/local/hadoop \ + HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop \ + YARN_CONF_DIR=/usr/local/hadoop/etc/hadoop \ + PATH=${PATH}:/usr/local/hadoop/bin + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + openssh-server \ + procps \ + rsync \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /run/sshd /root/.ssh # hadoop -RUN curl -s "https://www.apache.org/dyn/closer.lua/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz?action=download" | tar -xz -C /usr/local/ -RUN cd /usr/local && ln -s ./hadoop-$HADOOP_VERSION hadoop - -ENV HADOOP_PREFIX /usr/local/hadoop -ENV HADOOP_COMMON_HOME /usr/local/hadoop -ENV HADOOP_HDFS_HOME /usr/local/hadoop -ENV HADOOP_MAPRED_HOME /usr/local/hadoop -ENV HADOOP_YARN_HOME /usr/local/hadoop -ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop - -RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/lib/jvm/jre-1.7.0-openjdk\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh -RUN sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh +RUN case "$TARGETARCH" in \ + amd64) HADOOP_ARCHIVE="hadoop-$HADOOP_VERSION.tar.gz" ;; \ + arm64) HADOOP_ARCHIVE="hadoop-$HADOOP_VERSION-aarch64.tar.gz" ;; \ + *) echo "Unsupported architecture: $TARGETARCH" >&2; exit 1 ;; \ + esac \ + && HADOOP_URL="https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/$HADOOP_ARCHIVE" \ + && curl -fL "$HADOOP_URL" -o "/tmp/$HADOOP_ARCHIVE" \ + && curl -fL "$HADOOP_URL.sha512" -o "/tmp/$HADOOP_ARCHIVE.sha512" \ + && HADOOP_SHA512="$(sed -n 's/^SHA512 ([^)]*) = \([[:xdigit:]]\{128\}\)$/\1/p' "/tmp/$HADOOP_ARCHIVE.sha512")" \ + && test -n "$HADOOP_SHA512" \ + && echo "$HADOOP_SHA512 /tmp/$HADOOP_ARCHIVE" | sha512sum -c - \ + && tar -xzf "/tmp/$HADOOP_ARCHIVE" -C /usr/local/ \ + && rm "/tmp/$HADOOP_ARCHIVE" "/tmp/$HADOOP_ARCHIVE.sha512" \ + && ln -s /usr/local/hadoop-$HADOOP_VERSION $HADOOP_PREFIX + +RUN printf '\nexport JAVA_HOME=%s\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\nexport HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop\nexport HDFS_NAMENODE_USER=root\nexport HDFS_DATANODE_USER=root\nexport HDFS_SECONDARYNAMENODE_USER=root\nexport YARN_RESOURCEMANAGER_USER=root\nexport YARN_NODEMANAGER_USER=root\n' "$JAVA_HOME" \ + >> $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh RUN mkdir $HADOOP_PREFIX/input RUN cp $HADOOP_PREFIX/etc/hadoop/*.xml $HADOOP_PREFIX/input # hadoop configurations -ADD hdfs_conf/core-site.xml $HADOOP_PREFIX/etc/hadoop/core-site.xml -ADD hdfs_conf/hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml -ADD hdfs_conf/mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml -ADD hdfs_conf/yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml +COPY hdfs_conf/core-site.xml $HADOOP_PREFIX/etc/hadoop/core-site.xml +COPY hdfs_conf/hdfs-site.xml $HADOOP_PREFIX/etc/hadoop/hdfs-site.xml +COPY hdfs_conf/mapred-site.xml $HADOOP_PREFIX/etc/hadoop/mapred-site.xml +COPY hdfs_conf/yarn-site.xml $HADOOP_PREFIX/etc/hadoop/yarn-site.xml RUN mkdir /data/ RUN chmod 777 /data/ RUN $HADOOP_PREFIX/bin/hdfs namenode -format -RUN rm /usr/local/hadoop/lib/native/* -RUN curl -Ls http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-$HADOOP_VERSION.tar|tar -x -C /usr/local/hadoop/lib/native/ - -# install spark -RUN curl -s "https://www.apache.org/dyn/closer.lua/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE.tgz?action=download" | tar -xz -C /usr/local/ -RUN cd /usr/local && ln -s spark-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE spark -ENV SPARK_HOME /usr/local/spark - -ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop -ENV PATH $PATH:$SPARK_HOME/bin:$HADOOP_PREFIX/bin - # passwordless ssh -RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key -RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key +RUN ssh-keygen -A RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys -ADD ssh_config /root/.ssh/config +COPY ssh_config /root/.ssh/config RUN chmod 600 /root/.ssh/config RUN chown root:root /root/.ssh/config RUN chmod +x /usr/local/hadoop/etc/hadoop/*-env.sh # update boot script COPY entrypoint.sh /etc/entrypoint.sh -RUN chown root.root /etc/entrypoint.sh +RUN chown root:root /etc/entrypoint.sh RUN chmod 700 /etc/entrypoint.sh # Hdfs ports -EXPOSE 50010 50020 50070 50075 50090 +EXPOSE 50010 50020 50075 9868 9870 # Mapred ports EXPOSE 9000 9001 #Yarn ports diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh index 85b335d6d12..9a897d9b104 100755 --- a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh +++ b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/entrypoint.sh @@ -14,25 +14,37 @@ # See the License for the specific language governing permissions and # limitations under the License. -echo 'hadoop' |passwd root --stdin +set -euo pipefail -: ${HADOOP_PREFIX:=/usr/local/hadoop} +: "${HADOOP_PREFIX:=/usr/local/hadoop}" -$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh +. "$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh" -rm /tmp/*.pid +rm -f /tmp/*.pid # installing libraries if any - (resource urls added comma separated to the ACP system variable) -cd $HADOOP_PREFIX/share/hadoop/common ; for cp in ${ACP//,/ }; do echo == $cp; curl -LO $cp ; done; cd - +cd "$HADOOP_PREFIX/share/hadoop/common" +ACP_URLS="${ACP:-}" +for cp in ${ACP_URLS//,/ }; do + echo "== $cp" + curl -fLO -- "$cp" +done +cd - > /dev/null -cp $SPARK_HOME/conf/metrics.properties.template $SPARK_HOME/conf/metrics.properties +cp "$SPARK_HOME/conf/metrics.properties.template" "$SPARK_HOME/conf/metrics.properties" || true # start hadoop -service sshd start -$HADOOP_PREFIX/sbin/start-dfs.sh -$HADOOP_PREFIX/sbin/start-yarn.sh +service ssh start +"$HADOOP_PREFIX/sbin/start-dfs.sh" +"$HADOOP_PREFIX/sbin/start-yarn.sh" -$HADOOP_PREFIX/bin/hdfs dfsadmin -safemode leave && $HADOOP_PREFIX/bin/hdfs dfs -put $SPARK_HOME-$SPARK_VERSION-bin-hadoop$HADOOP_PROFILE/lib /spark +"$HADOOP_PREFIX/bin/hdfs" dfsadmin -safemode leave \ + && "$HADOOP_PREFIX/bin/hdfs" dfs -mkdir -p /spark +if ! "$HADOOP_PREFIX/bin/hdfs" dfs -test -e /spark/.jars-upload-complete; then + "$HADOOP_PREFIX/bin/hdfs" dfs -rm -r -f /spark/jars + "$HADOOP_PREFIX/bin/hdfs" dfs -put "$SPARK_HOME/jars" /spark + "$HADOOP_PREFIX/bin/hdfs" dfs -touchz /spark/.jars-upload-complete +fi # start spark export SPARK_MASTER_OPTS="-Dspark.driver.port=7001 -Dspark.fileserver.port=7002 @@ -46,14 +58,14 @@ export SPARK_WORKER_OPTS="-Dspark.driver.port=7001 -Dspark.fileserver.port=7002 export SPARK_MASTER_PORT=7077 -cd /usr/local/spark/sbin +cd "$SPARK_HOME/sbin" ./start-master.sh -./start-slave.sh spark://`hostname`:$SPARK_MASTER_PORT +./start-worker.sh "spark://$(hostname):$SPARK_MASTER_PORT" CMD=${1:-"exit 0"} if [[ "$CMD" == "-d" ]]; then - service sshd stop + service ssh stop /usr/sbin/sshd -D -d else /bin/bash -c "$*" diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/core-site.xml b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/core-site.xml index 87446337f9a..e460430190e 100644 --- a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/core-site.xml +++ b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/core-site.xml @@ -17,6 +17,6 @@ limitations under the License. fs.defaultFS - hdfs://0.0.0.0:9000 + hdfs://sparkmaster:9000