Skip to content
This repository was archived by the owner on Jul 23, 2024. It is now read-only.

Commit c3ae8e2

Browse files
author
Mariano Gonzalez
committed
Added support for HAWQ and PXF installation
1 parent ae2af74 commit c3ae8e2

8 files changed

Lines changed: 580 additions & 5 deletions

File tree

‎contrib/hawq-docker/Makefile‎

Lines changed: 223 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,13 @@ OS_VERSION := centos7
2525
# Do not use underscore "_" in CLUSTER_ID
2626
CLUSTER_ID := $(OS_VERSION)
2727
# Monut this local directory to /data in data container and share with other containers
28-
LOCAL :=
28+
LOCAL :=
2929
# networks used in docker
3030
NETWORK := $(CLUSTER_ID)_hawq_network
31+
HAWQ_HOME := "/data/hawq-devel"
32+
JAVA_TOOL_OPTIONS := -Dfile.encoding=UTF8
3133

32-
all:
34+
all:
3335
@echo " Usage:"
3436
@echo " To setup a build and test environment: make run"
3537
@echo " To start all containers: make start"
@@ -39,6 +41,17 @@ all:
3941
@echo ""
4042
@echo " To build images locally: make build"
4143
@echo " To pull latest images: make pull"
44+
@echo ""
45+
@echo " To build Hawq runtime: make hawq"
46+
@echo " To initialize Hawq on Namenode: make init-hawq"
47+
@echo " To start Hawq on Namenode: make start-hawq"
48+
@echo " To stop Hawq on Namenode: make stop-hawq"
49+
@echo " To check Hawq status on Namenode: make status-hawq"
50+
@echo " To build PXF runtime: make pxf"
51+
@echo " To initialize PXF on Namenode/Datanodes: make init-pxf"
52+
@echo " To start PXF on Namenode/Datanodes: make start-pxf"
53+
@echo " To stop PXF on on Namenode/Datanodes: make stop-hawq"
54+
@echo " To check PXF status on Namenode/Datanodes: make status-pxf"
4255

4356
build:
4457
@make -f $(THIS_MAKEFILE_PATH) build-hawq-dev-$(OS_VERSION)
@@ -227,3 +240,211 @@ distclean:
227240
docker network rm $(NETWORK) 2>&1 >/dev/null || true; \
228241
fi
229242
@echo "Distclean Done!"
243+
244+
hawq:
245+
@echo "Logging into ${CLUSTER_ID}-namenode container"
246+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
247+
docker exec \
248+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
249+
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
250+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --build"; \
251+
else \
252+
echo "${CLUSTER_ID}-namenode container does not exist!"; \
253+
fi
254+
255+
init-hawq:
256+
@echo "Make sure you have executed make hawq"
257+
@echo "Logging into ${CLUSTER_ID}-namenode container"
258+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
259+
docker exec \
260+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
261+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
262+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --init"; \
263+
else \
264+
echo "${CLUSTER_ID}-namenode container does not exist!"; \
265+
fi
266+
267+
start-hawq:
268+
@echo "Logging into ${CLUSTER_ID}-namenode container"
269+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
270+
docker exec \
271+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
272+
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
273+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --start"; \
274+
else \
275+
echo "${CLUSTER_ID}-namenode container does not exist!"; \
276+
fi
277+
278+
stop-hawq:
279+
@echo "Logging into ${CLUSTER_ID}-namenode container"
280+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
281+
docker exec \
282+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
283+
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
284+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --stop"; \
285+
else \
286+
echo "${CLUSTER_ID}-namenode container does not exist!"; \
287+
fi
288+
289+
status-hawq:
290+
@echo "Logging into ${CLUSTER_ID}-namenode container"
291+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
292+
docker exec \
293+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
294+
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
295+
-e "USER=gpadmin" \
296+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --status"; \
297+
else \
298+
echo "${CLUSTER_ID}-namenode container does not exist!"; \
299+
fi
300+
301+
pxf:
302+
@make -f $(THIS_MAKEFILE_PATH) pxf-namenode
303+
@i=1; \
304+
while [ $$i -le $(NDATANODES) ] ; do \
305+
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i pxf-datanode; \
306+
i=$$((i+1)); \
307+
done
308+
309+
pxf-namenode:
310+
@echo "Logging into ${CLUSTER_ID}-namenode container"
311+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
312+
docker exec \
313+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
314+
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
315+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --build"; \
316+
else \
317+
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
318+
fi
319+
320+
pxf-datanode:
321+
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
322+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
323+
docker exec \
324+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
325+
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
326+
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --build"; \
327+
else \
328+
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
329+
fi
330+
331+
init-pxf:
332+
@echo "Make sure you have executed make pxf"
333+
@make -f $(THIS_MAKEFILE_PATH) init-pxf-namenode
334+
@i=1; \
335+
while [ $$i -le $(NDATANODES) ] ; do \
336+
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i init-pxf-datanode; \
337+
i=$$((i+1)); \
338+
done
339+
340+
init-pxf-namenode:
341+
@echo "Logging into ${CLUSTER_ID}-namenode container"
342+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
343+
docker exec \
344+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
345+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
346+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --init"; \
347+
else \
348+
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
349+
fi
350+
351+
init-pxf-datanode:
352+
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
353+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
354+
docker exec \
355+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
356+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
357+
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --init"; \
358+
else \
359+
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
360+
fi
361+
362+
start-pxf:
363+
@make -f $(THIS_MAKEFILE_PATH) start-pxf-namenode
364+
@i=1; \
365+
while [ $$i -le $(NDATANODES) ] ; do \
366+
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i start-pxf-datanode; \
367+
i=$$((i+1)); \
368+
done
369+
370+
start-pxf-namenode:
371+
@echo "Logging into ${CLUSTER_ID}-namenode container"
372+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
373+
docker exec \
374+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
375+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
376+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --start"; \
377+
else \
378+
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
379+
fi
380+
381+
start-pxf-datanode:
382+
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
383+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
384+
docker exec \
385+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
386+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
387+
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --start"; \
388+
else \
389+
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
390+
fi
391+
392+
stop-pxf:
393+
@make -f $(THIS_MAKEFILE_PATH) stop-pxf-namenode
394+
@i=1; \
395+
while [ $$i -le $(NDATANODES) ] ; do \
396+
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i stop-pxf-datanode; \
397+
i=$$((i+1)); \
398+
done
399+
400+
stop-pxf-namenode:
401+
@echo "Logging into ${CLUSTER_ID}-namenode container"
402+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
403+
docker exec \
404+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
405+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
406+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --stop"; \
407+
else \
408+
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
409+
fi
410+
411+
stop-pxf-datanode:
412+
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
413+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
414+
docker exec \
415+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
416+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
417+
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --stop"; \
418+
else \
419+
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
420+
fi
421+
422+
status-pxf:
423+
@make -f $(THIS_MAKEFILE_PATH) status-pxf-namenode
424+
@i=1; \
425+
while [ $$i -le $(NDATANODES) ] ; do \
426+
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i status-pxf-datanode; \
427+
i=$$((i+1)); \
428+
done
429+
430+
status-pxf-namenode:
431+
@echo "Logging into ${CLUSTER_ID}-namenode container"
432+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
433+
docker exec \
434+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
435+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
436+
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh status"; \
437+
else \
438+
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
439+
fi
440+
441+
status-pxf-datanode:
442+
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
443+
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
444+
docker exec \
445+
-e "HAWQ_HOME=$(HAWQ_HOME)" \
446+
-e "NAMENODE=${CLUSTER_ID}-namenode" \
447+
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh status"; \
448+
else \
449+
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
450+
fi

‎contrib/hawq-docker/README.md‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,5 +123,14 @@ make pull
123123
To remove all containers: make distclean
124124
To build images locally: make build
125125
To pull latest images: make pull
126+
To build Hawq runtime: make hawq
127+
To initialize Hawq on Namenode: make init-hawq
128+
To start Hawq on Namenode: make start-hawq
129+
To stop Hawq on Namenode: make stop-hawq
130+
To check Hawq status on Namenode: make status-hawq
131+
To build PXF runtime: make pxf
132+
To initialize PXF on Namenode/Datanodes: make init-pxf
133+
To start PXF on Namenode/Datanodes: make start-pxf
134+
To stop PXF on on Namenode/Datanodes: make stop-hawq
135+
To check PXF status on Namenode/Datanodes: make status-hawq
126136
```
127-

‎contrib/hawq-docker/centos7-docker/hawq-dev/Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ RUN yum install -y epel-release && \
3030
openldap-devel protobuf-devel readline-devel net-snmp-devel apr-devel \
3131
libesmtp-devel python-pip json-c-devel \
3232
java-1.7.0-openjdk-devel lcov cmake \
33-
openssh-clients openssh-server perl-JSON && \
33+
openssh-clients openssh-server perl-JSON unzip && \
3434
yum clean all
3535

3636
RUN rpm -ivh --nodeps https://rpmfind.net/linux/centos/6.10/os/x86_64/Packages/bison-2.4.1-5.el6.x86_64.rpm

‎contrib/hawq-docker/centos7-docker/hawq-test/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ RUN ln -s /usr/hdp/current/hadoop-hdfs-namenode/../hadoop/sbin/hadoop-daemon.sh
3131
COPY conf/* /etc/hadoop/conf/
3232

3333
COPY entrypoint.sh /usr/bin/entrypoint.sh
34+
COPY service-hawq.sh /usr/bin/service-hawq.sh
35+
COPY service-pxf.sh /usr/bin/service-pxf.sh
3436
COPY start-hdfs.sh /usr/bin/start-hdfs.sh
3537

3638
USER gpadmin

‎contrib/hawq-docker/centos7-docker/hawq-test/conf/core-site.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,6 @@
1919
<configuration>
2020
<property>
2121
<name>fs.defaultFS</name>
22-
<value>hdfs://${hdfs.namenode}:8020</value>
22+
<value>hdfs://@hdfs.namenode@:8020</value>
2323
</property>
2424
</configuration>

‎contrib/hawq-docker/centos7-docker/hawq-test/entrypoint.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ if [ ! -f /etc/profile.d/hadoop.sh ]; then
2727
sudo chmod a+x /etc/profile.d/hadoop.sh
2828
fi
2929

30+
sudo chmod 777 /etc/hadoop/conf/core-site.xml
31+
sudo sed "s/@hdfs.namenode@/$NAMENODE/g" -i /etc/hadoop/conf/core-site.xml
32+
3033
sudo start-hdfs.sh
3134
sudo sysctl -p
3235

0 commit comments

Comments
 (0)