Skip to content

Commit 856875a

Browse files
Aggarwal-RaghavAggarwal-Raghav
committed
TEZ-4682: [Cloud] Tez AM docker image
1 parent ffd346b commit 856875a

5 files changed

Lines changed: 461 additions & 0 deletions

File tree

‎tez-dist/pom.xml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,38 @@
118118
</dependency>
119119
</dependencies>
120120
</profile>
121+
<profile>
122+
<id>docker</id>
123+
<build>
124+
<plugins>
125+
<plugin>
126+
<groupId>org.codehaus.mojo</groupId>
127+
<artifactId>exec-maven-plugin</artifactId>
128+
<executions>
129+
<execution>
130+
<id>build-docker-image</id>
131+
<phase>package</phase>
132+
<goals>
133+
<goal>exec</goal>
134+
</goals>
135+
<configuration>
136+
<executable>/bin/bash</executable>
137+
<arguments>
138+
<argument>${project.basedir}/src/docker/build-docker.sh</argument>
139+
<argument>-hadoop</argument>
140+
<argument>${hadoop.version}</argument>
141+
<argument>-tez</argument>
142+
<argument>${project.version}</argument>
143+
<argument>-repo</argument>
144+
<argument>apache</argument>
145+
</arguments>
146+
</configuration>
147+
</execution>
148+
</executions>
149+
</plugin>
150+
</plugins>
151+
</build>
152+
</profile>
121153
</profiles>
122154

123155
<build>

‎tez-dist/src/docker/Dockerfile‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one or more
3+
# contributor license agreements. See the NOTICE file distributed with
4+
# this work for additional information regarding copyright ownership.
5+
# The ASF licenses this file to You under the Apache License, Version 2.0
6+
# (the "License"); you may not use this file except in compliance with
7+
# the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
#
17+
18+
ARG BUILD_ENV=unarchive
19+
20+
FROM ubuntu AS unarchive
21+
ONBUILD COPY hadoop-*.tar.gz /opt
22+
# UPDATED: Matches "tez-1.0.0-SNAPSHOT.tar.gz" pattern
23+
ONBUILD COPY tez-*.tar.gz /opt
24+
25+
FROM ${BUILD_ENV} AS env
26+
ARG HADOOP_VERSION
27+
ARG TEZ_VERSION
28+
29+
RUN mkdir -p /opt/hadoop \
30+
&& tar -xzv \
31+
--exclude="hadoop-$HADOOP_VERSION/share/doc" \
32+
--exclude="*/jdiff" \
33+
--exclude="*/sources" \
34+
--exclude="*tests.jar" \
35+
--exclude="*/webapps" \
36+
-f /opt/hadoop-$HADOOP_VERSION.tar.gz \
37+
-C /opt/hadoop --strip-components 1 \
38+
&& mkdir -p /opt/tez \
39+
&& tar -xzv \
40+
-f /opt/tez-$TEZ_VERSION.tar.gz \
41+
-C /opt/tez \
42+
&& rm -rf /opt/hadoop-$HADOOP_VERSION.tar.gz /opt/tez-$TEZ_VERSION.tar.gz
43+
44+
FROM eclipse-temurin:21.0.3_9-jre-ubi9-minimal AS run
45+
46+
ARG UID=1000
47+
ARG HADOOP_VERSION
48+
ARG TEZ_VERSION
49+
50+
# Install dependencies
51+
RUN set -ex; \
52+
microdnf update -y; \
53+
microdnf -y install procps gettext findutils; \
54+
microdnf clean all; \
55+
useradd --no-create-home -s /sbin/nologin -c "" --uid $UID tez
56+
57+
# Set necessary environment variables
58+
ENV HADOOP_HOME=/opt/hadoop \
59+
TEZ_HOME=/opt/tez \
60+
TEZ_CONF_DIR=/opt/tez/conf \
61+
HADOOP_CONF_DIR=/opt/tez/conf
62+
63+
ENV PATH=$TEZ_HOME/bin:$HADOOP_HOME/bin:$PATH
64+
65+
COPY --from=env --chown=tez /opt/hadoop $HADOOP_HOME
66+
# UPDATED: Copy from the normalized directory name created in 'env' stage
67+
COPY --from=env --chown=tez /opt/tez $TEZ_HOME
68+
69+
RUN mkdir -p $TEZ_CONF_DIR && chown tez:tez $TEZ_CONF_DIR
70+
71+
COPY --chown=tez entrypoint.sh /
72+
COPY --chown=tez conf $TEZ_CONF_DIR
73+
74+
# Create Extension Point Directory
75+
RUN mkdir -p /opt/tez/plugins && chown tez:tez /opt/tez/plugins && chmod 755 /opt/tez/plugins
76+
77+
RUN chmod +x /entrypoint.sh
78+
79+
USER tez
80+
WORKDIR $TEZ_HOME
81+
82+
# Expose AM RPC Ports
83+
EXPOSE 10001 10002 10003 8042 2181
84+
85+
ENTRYPOINT ["/entrypoint.sh"]

‎tez-dist/src/docker/build-docker.sh‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one or more
5+
# contributor license agreements. See the NOTICE file distributed with
6+
# this work for additional information regarding copyright ownership.
7+
# The ASF licenses this file to You under the Apache License, Version 2.0
8+
# (the "License"); you may not use this file except in compliance with
9+
# the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
#
19+
20+
set -xeou pipefail
21+
22+
HADOOP_VERSION=
23+
TEZ_VERSION=
24+
REPO=
25+
26+
usage() {
27+
cat <<EOF 1>&2
28+
Usage: $0 [-h] [-hadoop <Hadoop version>] [-tez <Tez version>] [-repo <Docker repo>]
29+
Build the Apache Tez AM Docker image
30+
-help Display help
31+
-hadoop Build image with the specified Hadoop version
32+
-tez Build image with the specified Tez version
33+
-repo Docker repository
34+
EOF
35+
}
36+
37+
while [ $# -gt 0 ]; do
38+
case "$1" in
39+
-h)
40+
usage
41+
exit 0
42+
;;
43+
-hadoop)
44+
shift
45+
HADOOP_VERSION=$1
46+
shift
47+
;;
48+
-tez)
49+
shift
50+
TEZ_VERSION=$1
51+
shift
52+
;;
53+
-repo)
54+
shift
55+
REPO=$1
56+
shift
57+
;;
58+
*)
59+
shift
60+
;;
61+
esac
62+
done
63+
64+
SCRIPT_DIR=$(
65+
cd "$(dirname "$0")"
66+
pwd
67+
)
68+
69+
DIST_DIR=${DIST_DIR:-"$SCRIPT_DIR/../.."}
70+
PROJECT_ROOT=${PROJECT_ROOT:-"$SCRIPT_DIR/../../.."}
71+
72+
repo=${REPO:-apache}
73+
WORK_DIR="$(mktemp -d)"
74+
CACHE_DIR="$SCRIPT_DIR/cache"
75+
mkdir -p "$CACHE_DIR"
76+
77+
# Defaults Hadoop and Tez versions from pom.xml if not provided
78+
HADOOP_VERSION=${HADOOP_VERSION:-$(mvn -f "$PROJECT_ROOT/pom.xml" -q help:evaluate -Dexpression=hadoop.version -DforceStdout)}
79+
TEZ_VERSION=${TEZ_VERSION:-$(mvn -f "$PROJECT_ROOT/pom.xml" -q help:evaluate -Dexpression=project.version -DforceStdout)}
80+
81+
######################
82+
# HADOOP FETCH LOGIC #
83+
######################
84+
HADOOP_FILE_NAME="hadoop-$HADOOP_VERSION.tar.gz"
85+
HADOOP_URL=${HADOOP_URL:-"https://archive.apache.org/dist/hadoop/core/hadoop-$HADOOP_VERSION/$HADOOP_FILE_NAME"}
86+
if [ ! -f "$CACHE_DIR/$HADOOP_FILE_NAME" ]; then
87+
echo "Downloading Hadoop from $HADOOP_URL..."
88+
if ! curl --fail -L "$HADOOP_URL" -o "$CACHE_DIR/$HADOOP_FILE_NAME.tmp"; then
89+
echo "Fail to download Hadoop, exiting...."
90+
exit 1
91+
fi
92+
mv "$CACHE_DIR/$HADOOP_FILE_NAME.tmp" "$CACHE_DIR/$HADOOP_FILE_NAME"
93+
fi
94+
95+
#####################################
96+
# Pick tez tarball from local build #
97+
#####################################
98+
TEZ_FILE_NAME="tez-$TEZ_VERSION.tar.gz"
99+
LOCAL_DIST_PATH="$DIST_DIR/target/$TEZ_FILE_NAME"
100+
101+
if [ -f "$LOCAL_DIST_PATH" ]; then
102+
echo "--> Found local Tez build artifact at: $LOCAL_DIST_PATH"
103+
cp "$LOCAL_DIST_PATH" "$WORK_DIR/"
104+
else
105+
echo "--> Error: Local Tez artifact not found at $LOCAL_DIST_PATH"
106+
echo "--> Please build the project first (e.g., mvn clean install -DskipTests)."
107+
exit 1
108+
fi
109+
110+
# -------------------------------------------------------------------------
111+
# BUILD CONTEXT PREPARATION
112+
# -------------------------------------------------------------------------
113+
cp "$CACHE_DIR/$HADOOP_FILE_NAME" "$WORK_DIR/"
114+
cp -R "$SCRIPT_DIR/conf" "$WORK_DIR/" 2>/dev/null || mkdir -p "$WORK_DIR/conf"
115+
cp "$SCRIPT_DIR/entrypoint.sh" "$WORK_DIR/"
116+
cp "$SCRIPT_DIR/Dockerfile" "$WORK_DIR/"
117+
118+
echo "Building Docker image..."
119+
docker build \
120+
"$WORK_DIR" \
121+
-f "$WORK_DIR/Dockerfile" \
122+
-t "$repo/tez-am:$TEZ_VERSION" \
123+
--build-arg "BUILD_ENV=unarchive" \
124+
--build-arg "HADOOP_VERSION=$HADOOP_VERSION" \
125+
--build-arg "TEZ_VERSION=$TEZ_VERSION"
126+
127+
rm -r "${WORK_DIR}"
128+
echo "Docker image $repo/tez-am:$TEZ_VERSION built successfully."

‎tez-dist/src/docker/conf/tez-site.xml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
<?xml version="1.0"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
<!--
4+
Licensed to the Apache Software Foundation (ASF) under one or more
5+
contributor license agreements. See the NOTICE file distributed with
6+
this work for additional information regarding copyright ownership.
7+
The ASF licenses this file to You under the Apache License, Version 2.0
8+
(the "License"); you may not use this file except in compliance with
9+
the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
20+
<configuration>
21+
<property>
22+
<name>tez.am.client.am.port-range</name>
23+
<value>10001-10003</value>
24+
</property>
25+
26+
<property>
27+
<name>tez.am.resource.memory.mb</name>
28+
<value>1024</value>
29+
</property>
30+
31+
<property>
32+
<name>tez.framework.mode</name>
33+
<value>STANDALONE_ZOOKEEPER</value>
34+
</property>
35+
36+
<property>
37+
<name>tez.am.zookeeper.quorum</name>
38+
<value>host.docker.internal:2181</value>
39+
</property>
40+
41+
<property>
42+
<name>tez.am.log.level</name>
43+
<value>DEBUG</value>
44+
</property>
45+
46+
<property>
47+
<name>tez.am.mode.session</name>
48+
<value>true</value>
49+
</property>
50+
51+
52+
</configuration>

0 commit comments

Comments
 (0)