Test fg reader hive3

docker/compose/docker-compose_hadoop310_hive312_spark321_mac_aarch64.yml

@@ -0,0 +1,319 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: "3.3"
+
+services:
+
+ namenode:
+ image: apachehudi/hudi-hadoop_3.1.0-namenode:latest
+ hostname: namenode
+ container_name: namenode
+ environment:
+ - CLUSTER_NAME=hudi_hadoop310_hive312_spark321
+ ports:
+ - "9870:9870"
+ - "8020:8020"
+ env_file:
+ - ./hadoop.env
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://namenode:9870"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+
+ datanode1:
+ image: apachehudi/hudi-hadoop_3.1.0-datanode:latest
+ container_name: datanode1
+ hostname: datanode1
+ environment:
+ - CLUSTER_NAME=hudi_hadoop310_hive312_spark321
+ env_file:
+ - ./hadoop.env
+ ports:
+ - "50075:50075"
+ - "50010:50010"
+ links:
+ - "namenode"
+ - "historyserver"
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://datanode1:50075"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ depends_on:
+ - namenode
+
+ historyserver:
+ image: apachehudi/hudi-hadoop_3.1.0-history:latest
+ hostname: historyserver
+ container_name: historyserver
+ environment:
+ - CLUSTER_NAME=hudi_hadoop310_hive312_spark321
+ depends_on:
+ - "namenode"
+ links:
+ - "namenode"
+ ports:
+ - "58188:8188"
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://historyserver:8188"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ env_file:
+ - ./hadoop.env
+ volumes:
+ - historyserver:/hadoop/yarn/timeline
+
+ hive-metastore-postgresql:
+ image: bde2020/hive-metastore-postgresql:3.1.0
+ volumes:
+ - hive-metastore-postgresql:/var/lib/postgresql
+ hostname: hive-metastore-postgresql
+ container_name: hive-metastore-postgresql
+
+ hivemetastore:
+ image: apachehudi/hudi-hadoop_3.1.0-hive_3.1.2:latest
+ hostname: hivemetastore
+ container_name: hivemetastore
+ links:
+ - "hive-metastore-postgresql"
+ - "namenode"
+ env_file:
+ - ./hadoop.env
+ command: /opt/hive/bin/hive --service metastore
+ environment:
+ SERVICE_PRECONDITION: "namenode:9870 hive-metastore-postgresql:5432"
+ ports:
+ - "9083:9083"
+ healthcheck:
+ test: ["CMD", "nc", "-z", "hivemetastore", "9083"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ depends_on:
+ - "hive-metastore-postgresql"
+ - "namenode"
+
+ hiveserver:
+ image: apachehudi/hudi-hadoop_3.1.0-hive_3.1.2:latest
+ hostname: hiveserver
+ container_name: hiveserver
+ env_file:
+ - ./hadoop.env
+ environment:
+ SERVICE_PRECONDITION: "hivemetastore:9083"
+ JAVA_TOOL_OPTIONS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005"
+ ports:
+ - "10000:10000"
+ # JVM debugging port
+ - "64757:5005"
+ depends_on:
+ - "hivemetastore"
+ links:
+ - "hivemetastore"
+ - "hive-metastore-postgresql"
+ - "namenode"
+ volumes:
+ - ${HUDI_WS}:/var/hoodie/ws
+ - /Users/jon/Desktop/hiveWorkload:/var/hiveWorkload
+
+ sparkmaster:
+ image: apachehudi/hudi-hadoop_3.1.0-hive_3.1.2-sparkmaster_3.2.1:latest
+ hostname: sparkmaster
+ container_name: sparkmaster
+ env_file:
+ - ./hadoop.env
+ ports:
+ - "8080:8080"
+ - "7077:7077"
+ environment:
+ - INIT_DAEMON_STEP=setup_spark
+ links:
+ - "hivemetastore"
+ - "hiveserver"
+ - "hive-metastore-postgresql"
+ - "namenode"
+
+ spark-worker-1:
+ image: apachehudi/hudi-hadoop_3.1.0-hive_3.1.2-sparkworker_3.2.1:latest
+ hostname: spark-worker-1
+ container_name: spark-worker-1
+ env_file:
+ - ./hadoop.env
+ depends_on:
+ - sparkmaster
+ ports:
+ - "8081:8081"
+ environment:
+ - "SPARK_MASTER=spark://sparkmaster:7077"
+ links:
+ - "hivemetastore"
+ - "hiveserver"
+ - "hive-metastore-postgresql"
+ - "namenode"
+
+ zookeeper:
+ image: 'arm64v8/zookeeper:3.4.12'
+ platform: linux/arm64
+ hostname: zookeeper
+ container_name: zookeeper
+ ports:
+ - "2181:2181"
+ environment:
+ - ALLOW_ANONYMOUS_LOGIN=yes
+
+ kafka:
+ image: 'wurstmeister/kafka:2.12-2.0.1'
+ platform: linux/arm64
+ hostname: kafkabroker
+ container_name: kafkabroker
+ ports:
+ - "9092:9092"
+ environment:
+ - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+ - ALLOW_PLAINTEXT_LISTENER=yes
+ - KAFKA_ADVERTISED_HOST_NAME=kafkabroker
+
+# presto-coordinator-1:
+# container_name: presto-coordinator-1
+# hostname: presto-coordinator-1
+# image: apachehudi/hudi-hadoop_3.1.0-prestobase_0.271:latest
+# ports:
+# - '8090:8090'
+# environment:
+# - PRESTO_JVM_MAX_HEAP=512M
+# - PRESTO_QUERY_MAX_MEMORY=1GB
+# - PRESTO_QUERY_MAX_MEMORY_PER_NODE=256MB
+# - PRESTO_QUERY_MAX_TOTAL_MEMORY_PER_NODE=384MB
+# - PRESTO_MEMORY_HEAP_HEADROOM_PER_NODE=100MB
+# - TERM=xterm
+# links:
+# - "hivemetastore"
+# volumes:
+# - ${HUDI_WS}:/var/hoodie/ws
+# command: coordinator
+#
+# presto-worker-1:
+# container_name: presto-worker-1
+# hostname: presto-worker-1
+# image: apachehudi/hudi-hadoop_3.1.0-prestobase_0.271:latest
+# depends_on: [ "presto-coordinator-1" ]
+# environment:
+# - PRESTO_JVM_MAX_HEAP=512M
+# - PRESTO_QUERY_MAX_MEMORY=1GB
+# - PRESTO_QUERY_MAX_MEMORY_PER_NODE=256MB
+# - PRESTO_QUERY_MAX_TOTAL_MEMORY_PER_NODE=384MB
+# - PRESTO_MEMORY_HEAP_HEADROOM_PER_NODE=100MB
+# - TERM=xterm
+# links:
+# - "hivemetastore"
+# - "hiveserver"
+# - "hive-metastore-postgresql"
+# - "namenode"
+# volumes:
+# - ${HUDI_WS}:/var/hoodie/ws
+# command: worker
+#
+# trino-coordinator-1:
+# container_name: trino-coordinator-1
+# hostname: trino-coordinator-1
+# image: apachehudi/hudi-hadoop_3.1.0-trinocoordinator_368:latest
+# ports:
+# - '8091:8091'
+# links:
+# - "hivemetastore"
+# volumes:
+# - ${HUDI_WS}:/var/hoodie/ws
+# command: http://trino-coordinator-1:8091 trino-coordinator-1
+#
+# trino-worker-1:
+# container_name: trino-worker-1
+# hostname: trino-worker-1
+# image: apachehudi/hudi-hadoop_3.1.0-trinoworker_368:latest
+# depends_on: [ "trino-coordinator-1" ]
+# ports:
+# - '8092:8092'
+# links:
+# - "hivemetastore"
+# - "hiveserver"
+# - "hive-metastore-postgresql"
+# - "namenode"
+# volumes:
+# - ${HUDI_WS}:/var/hoodie/ws
+# command: http://trino-coordinator-1:8091 trino-worker-1
+#
+# graphite:
+# container_name: graphite
+# hostname: graphite
+# image: graphiteapp/graphite-statsd
+# ports:
+# - 80:80
+# - 2003-2004:2003-2004
+# - 8126:8126
+
+ adhoc-1:
+ image: apachehudi/hudi-hadoop_3.1.0-hive_3.1.2-sparkadhoc_3.2.1:latest
+ hostname: adhoc-1
+ container_name: adhoc-1
+ env_file:
+ - ./hadoop.env
+ depends_on:
+ - sparkmaster
+ ports:
+ - '4040:4040'
+ environment:
+ - "SPARK_MASTER=spark://sparkmaster:7077"
+ links:
+ - "hivemetastore"
+ - "hiveserver"
+ - "hive-metastore-postgresql"
+ - "namenode"
+ #- "presto-coordinator-1"
+ #- "trino-coordinator-1"
+ volumes:
+ - ${HUDI_WS}:/var/hoodie/ws
+ - /Users/jon/Desktop/hiveWorkload:/var/hiveWorkload
+
+ adhoc-2:
+ image: apachehudi/hudi-hadoop_3.1.0-hive_3.1.2-sparkadhoc_3.2.1:latest
+ hostname: adhoc-2
+ container_name: adhoc-2
+ env_file:
+ - ./hadoop.env
+ depends_on:
+ - sparkmaster
+ environment:
+ - "SPARK_MASTER=spark://sparkmaster:7077"
+ links:
+ - "hivemetastore"
+ - "hiveserver"
+ - "hive-metastore-postgresql"
+ - "namenode"
+ #- "presto-coordinator-1"
+ #- "trino-coordinator-1"
+ volumes:
+ - ${HUDI_WS}:/var/hoodie/ws
+ - /Users/jon/Desktop/hiveWorkload:/var/hiveWorkload
+
+volumes:
+ namenode:
+ historyserver:
+ hive-metastore-postgresql:
+
+networks:
+ default:
+ name: hudi-network

docker/setup_demo.sh

@@ -23,6 +23,11 @@ COMPOSE_FILE_NAME="docker-compose_hadoop284_hive233_spark244.yml"
 if [ "$HUDI_DEMO_ENV" = "--mac-aarch64" ]; then
  COMPOSE_FILE_NAME="docker-compose_hadoop284_hive233_spark244_mac_aarch64.yml"
 fi
+
+if [ "$HUDI_DEMO_ENV" = "--hive3" ]; then
+ COMPOSE_FILE_NAME="docker-compose_hadoop310_hive312_spark321_mac_aarch64.yml"
+fi
+
 # restart cluster
 HUDI_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/compose/${COMPOSE_FILE_NAME} down
 if [ "$HUDI_DEMO_ENV" != "dev" ]; then

docker/stop_demo.sh

@@ -24,6 +24,11 @@ COMPOSE_FILE_NAME="docker-compose_hadoop284_hive233_spark244.yml"
 if [ "$HUDI_DEMO_ENV" = "--mac-aarch64" ]; then
  COMPOSE_FILE_NAME="docker-compose_hadoop284_hive233_spark244_mac_aarch64.yml"
 fi
+
+if [ "$HUDI_DEMO_ENV" = "--hive3" ]; then
+ COMPOSE_FILE_NAME="docker-compose_hadoop310_hive312_spark321_mac_aarch64.yml"
+fi
+
 # shut down cluster
 HUDI_WS=${WS_ROOT} docker-compose -f ${SCRIPT_PATH}/compose/${COMPOSE_FILE_NAME} down