Skip to content
This repository has been archived by the owner on Jul 23, 2024. It is now read-only.

HAWQ-1658. Easy Hawq and PXF initialization #1396

Closed
wants to merge 1 commit into from
Closed

HAWQ-1658. Easy Hawq and PXF initialization #1396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 237 additions & 5 deletions contrib/hawq-docker/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,22 +25,36 @@ OS_VERSION := centos7
# Do not use underscore "_" in CLUSTER_ID
CLUSTER_ID := $(OS_VERSION)
# Monut this local directory to /data in data container and share with other containers
LOCAL :=
LOCAL :=
# networks used in docker
NETWORK := $(CLUSTER_ID)_hawq_network
HAWQ_HOME := "/data/hawq-devel"
PXF_CLASSPATH_TEMPLATE = "hdp"
JAVA_TOOL_OPTIONS := -Dfile.encoding=UTF8

all:
all:
eschizoid marked this conversation as resolved.
Show resolved Hide resolved
@echo " Usage:"
@echo " To setup a build and test environment: make run"
@echo " To start all containers: make start"
@echo " To stop all containers: make stop"
@echo " To remove hdfs containers: make clean"
@echo " To remove all containers: make distclean"
@echo ""
@echo " To build images locally: make build"
@echo " To build images locally: make build-image"
@echo " To pull latest images: make pull"
@echo ""
@echo " To build Hawq runtime: make build-hawq"
@echo " To initialize Hawq on Namenode: make init-hawq"
@echo " To start Hawq on Namenode: make start-hawq"
@echo " To stop Hawq on Namenode: make stop-hawq"
@echo " To check Hawq status on Namenode: make status-hawq"
@echo " To build PXF runtime: make build-pxf"
@echo " To initialize PXF on Namenode/Datanodes: make init-pxf"
@echo " To start PXF on Namenode/Datanodes: make start-pxf"
@echo " To stop PXF on on Namenode/Datanodes: make stop-hawq"
@echo " To check PXF status on Namenode/Datanodes: make status-pxf"

build:
build-image:
@make -f $(THIS_MAKEFILE_PATH) build-hawq-dev-$(OS_VERSION)
@make -f $(THIS_MAKEFILE_PATH) build-hawq-test-$(OS_VERSION)
@echo "Build Images Done!"
Expand All @@ -51,7 +65,10 @@ build-hawq-dev-$(OS_VERSION): $(TOP_DIR)/$(OS_VERSION)-docker/hawq-dev/Dockerfil

build-hawq-test-$(OS_VERSION): $(TOP_DIR)/$(OS_VERSION)-docker/hawq-test/Dockerfile
@echo build hawq-test:$(OS_VERSION) image
docker build -t hawq/hawq-test:$(OS_VERSION) $(TOP_DIR)/$(OS_VERSION)-docker/hawq-test/
docker build \
--build-arg=PXF_CLASSPATH_TEMPLATE="`cat ../../pxf/pxf-service/src/configs/templates/pxf-private-${PXF_CLASSPATH_TEMPLATE}.classpath.template`" \
Copy link
Contributor Author

@eschizoid eschizoid Sep 27, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might look like a hack, but I found it handy for loading files from a different docker build context. Happy to change it if there is a better way of doing this.

--build-arg=PXF_LOG4J_PROPERTIES="`cat ../../pxf/pxf-service/src/main/resources/pxf-log4j.properties`" \
-t hawq/hawq-test:$(OS_VERSION) $(TOP_DIR)/$(OS_VERSION)-docker/hawq-test/

create-data-container:
@echo create ${CLUSTER_ID}-data container
Expand Down Expand Up @@ -227,3 +244,218 @@ distclean:
docker network rm $(NETWORK) 2>&1 >/dev/null || true; \
fi
@echo "Distclean Done!"

build-hawq:
@echo "Make sure you have executed make build-image"
@echo "Make sure you have executed make run"
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --build"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!"; \
fi

init-hawq:
@echo "Make sure you have executed make build-hawq"
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --init"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!"; \
fi

start-hawq:
@echo "Make sure you have executed make init-hawq"
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --start"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!"; \
fi

stop-hawq:
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --stop"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!"; \
fi

status-hawq:
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
-e "USER=gpadmin" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-hawq.sh --status"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!"; \
fi

build-pxf:
@echo "Make sure you have executed make build-image"
@echo "Make sure you have executed make run"
@make -f $(THIS_MAKEFILE_PATH) pxf-namenode
@i=1; \
while [ $$i -le $(NDATANODES) ] ; do \
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i pxf-datanode; \
i=$$((i+1)); \
done

pxf-namenode:
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
-e "PXF_CLASSPATH_TEMPLATE=$(PXF_CLASSPATH_TEMPLATE)" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --build"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
fi

pxf-datanode:
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "JAVA_TOOL_OPTIONS=$(JAVA_TOOL_OPTIONS)" \
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --build"; \
else \
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
fi

init-pxf:
@echo "Make sure you have executed make build-pxf"
@make -f $(THIS_MAKEFILE_PATH) init-pxf-namenode
@i=1; \
while [ $$i -le $(NDATANODES) ] ; do \
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i init-pxf-datanode; \
i=$$((i+1)); \
done

init-pxf-namenode:
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --init"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
fi

init-pxf-datanode:
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --init"; \
else \
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
fi

start-pxf:
@echo "Make sure you have executed make init-pxf"
@make -f $(THIS_MAKEFILE_PATH) start-pxf-namenode
@i=1; \
while [ $$i -le $(NDATANODES) ] ; do \
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i start-pxf-datanode; \
i=$$((i+1)); \
done

start-pxf-namenode:
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --start"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
fi

start-pxf-datanode:
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --start"; \
else \
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
fi

stop-pxf:
@make -f $(THIS_MAKEFILE_PATH) stop-pxf-namenode
@i=1; \
while [ $$i -le $(NDATANODES) ] ; do \
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i stop-pxf-datanode; \
i=$$((i+1)); \
done

stop-pxf-namenode:
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --stop"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
fi

stop-pxf-datanode:
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --stop"; \
else \
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
fi

status-pxf:
@make -f $(THIS_MAKEFILE_PATH) status-pxf-namenode
@i=1; \
while [ $$i -le $(NDATANODES) ] ; do \
make -f $(THIS_MAKEFILE_PATH) CUR_DATANODE=$$i status-pxf-datanode; \
i=$$((i+1)); \
done

status-pxf-namenode:
@echo "Logging into ${CLUSTER_ID}-namenode container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-namenode" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-namenode /bin/bash -c "service-pxf.sh --status"; \
else \
echo "${CLUSTER_ID}-namenode container does not exist!" && exit 1; \
fi

status-pxf-datanode:
@echo "Logging into ${CLUSTER_ID}-datanode$(CUR_DATANODE) container"
@if [ ! -z "`docker ps -a --filter="name=${CLUSTER_ID}-datanode$(CUR_DATANODE)" | grep -v CONTAINER`" ]; then \
docker exec \
-e "HAWQ_HOME=$(HAWQ_HOME)" \
-e "NAMENODE=${CLUSTER_ID}-namenode" \
-u gpadmin --privileged -it ${CLUSTER_ID}-datanode$(CUR_DATANODE) /bin/bash -c "service-pxf.sh --status"; \
else \
echo "${CLUSTER_ID}-datanode$(CUR_DATANODE) container does not exist!" && exit 1; \
fi
13 changes: 11 additions & 2 deletions contrib/hawq-docker/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,16 @@ make pull
To stop all containers: make stop
To remove hdfs containers: make clean
To remove all containers: make distclean
To build images locally: make build
To build images locally: make build-image
To pull latest images: make pull
To build Hawq runtime: make build-hawq
To initialize Hawq on Namenode: make init-hawq
To start Hawq on Namenode: make start-hawq
To stop Hawq on Namenode: make stop-hawq
To check Hawq status on Namenode: make status-hawq
To build PXF runtime: make build-pxf
To initialize PXF on Namenode/Datanodes: make init-pxf
To start PXF on Namenode/Datanodes: make start-pxf
To stop PXF on on Namenode/Datanodes: make stop-hawq
To check PXF status on Namenode/Datanodes: make status-hawq
```

2 changes: 1 addition & 1 deletion contrib/hawq-docker/centos7-docker/hawq-dev/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ RUN yum install -y epel-release && \
openldap-devel protobuf-devel readline-devel net-snmp-devel apr-devel \
libesmtp-devel python-pip json-c-devel \
java-1.7.0-openjdk-devel lcov cmake \
openssh-clients openssh-server perl-JSON && \
openssh-clients openssh-server perl-JSON unzip && \
yum clean all

RUN rpm -ivh --nodeps https://rpmfind.net/linux/centos/6.10/os/x86_64/Packages/bison-2.4.1-5.el6.x86_64.rpm
Expand Down
10 changes: 10 additions & 0 deletions contrib/hawq-docker/centos7-docker/hawq-test/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,26 @@ MAINTAINER Richard Guo <[email protected]>

USER root

ARG PXF_CLASSPATH_TEMPLATE
ARG PXF_LOG4J_PROPERTIES

## install HDP 2.5.0
RUN curl -L "http://public-repo-1.hortonworks.com/HDP/centos7/2.x/updates/2.5.0.0/hdp.repo" -o /etc/yum.repos.d/hdp.repo && \
yum install -y hadoop hadoop-hdfs hadoop-libhdfs hadoop-yarn hadoop-mapreduce hadoop-client hdp-select && \
yum clean all

RUN ln -s /usr/hdp/current/hadoop-hdfs-namenode/../hadoop/sbin/hadoop-daemon.sh /usr/bin/hadoop-daemon.sh

RUN touch /tmp/pxf-private.classpath && \
touch /tmp/log4j.properties && \
echo "$PXF_CLASSPATH_TEMPLATE" > /tmp/pxf-private.classpath && \
echo "$PXF_LOG4J_PROPERTIES" > /tmp/pxf-log4j.properties

COPY conf/* /etc/hadoop/conf/

COPY entrypoint.sh /usr/bin/entrypoint.sh
COPY service-hawq.sh /usr/bin/service-hawq.sh
COPY service-pxf.sh /usr/bin/service-pxf.sh
COPY start-hdfs.sh /usr/bin/start-hdfs.sh

USER gpadmin
Expand Down
2 changes: 1 addition & 1 deletion contrib/hawq-docker/centos7-docker/hawq-test/conf/core-site.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://${hdfs.namenode}:8020</value>
<value>hdfs://@hdfs.namenode@:8020</value>
</property>
</configuration>
3 changes: 3 additions & 0 deletions contrib/hawq-docker/centos7-docker/hawq-test/entrypoint.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ if [ ! -f /etc/profile.d/hadoop.sh ]; then
sudo chmod a+x /etc/profile.d/hadoop.sh
fi

sudo chmod 777 /etc/hadoop/conf/core-site.xml
sudo sed "s/@hdfs.namenode@/$NAMENODE/g" -i /etc/hadoop/conf/core-site.xml

sudo start-hdfs.sh
sudo sysctl -p

Expand Down
Loading