Updated to start Spark Connect on the Spark master container

Signed-off-by: Norman Jordan <[email protected]>

docker/integ-test/.env

@@ -4,6 +4,7 @@ DASHBOARDS_VERSION=latest
 MASTER_UI_PORT=8080
 MASTER_PORT=7077
 UI_PORT=4040
+SPARK_CONNECT_PORT=15002
 PPL_JAR=../../ppl-spark-integration/target/scala-2.12/ppl-spark-integration-assembly-0.7.0-SNAPSHOT.jar
 FLINT_JAR=../../flint-spark-integration/target/scala-2.12/flint-spark-integration-assembly-0.7.0-SNAPSHOT.jar
 OPENSEARCH_NODE_MEMORY=512m

docker/integ-test/docker-compose.yml

@@ -6,6 +6,8 @@ services:
       - "${MASTER_UI_PORT:-8080}:8080"
       - "${MASTER_PORT:-7077}:7077"
       - "${UI_PORT:-4040}:4040"
+      - "${SPARK_CONNECT_PORT}:15002"
+    entrypoint: /opt/bitnami/scripts/spark/master-entrypoint.sh
     environment:
       - SPARK_MODE=master
       - SPARK_RPC_AUTHENTICATION_ENABLED=no
@@ -14,6 +16,9 @@ services:
       - SPARK_SSL_ENABLED=no
       - SPARK_PUBLIC_DNS=localhost
     volumes:
+      - type: bind
+        source: ./spark-master-entrypoint.sh
+        target: /opt/bitnami/scripts/spark/master-entrypoint.sh
       - type: bind
         source: ./spark-defaults.conf
         target: /opt/bitnami/spark/conf/spark-defaults.conf

docker/integ-test/spark-master-entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+function start_spark_connect() {
+  sc_version=$(ls -1 /opt/bitnami/spark/jars/spark-core_*.jar | sed -e 's/^.*\/spark-core_//' -e 's/\.jar$//' -e 's/-/:/')
+
+  attempt=1
+  while [ -e "/tmp/spark_master_running" -a "$attempt" -le 10 ]; do
+    sleep 1
+    /opt/bitnami/spark/sbin/start-connect-server.sh --master spark://spark:7077 --packages org.apache.spark:spark-connect_${sc_version}
+    attempt=$(($attempt+1))
+  done
+}
+
+touch /tmp/spark_master_running
+start_spark_connect &
+/opt/bitnami/scripts/spark/entrypoint.sh /opt/bitnami/scripts/spark/run.sh
+rm /tmp/spark_master_running