From 418ee7e67662db452a33d56bed27e6e1a688f619 Mon Sep 17 00:00:00 2001
From: Norman Jordan <norman.jordan@improving.com>
Date: Wed, 11 Dec 2024 14:35:35 -0800
Subject: [PATCH] Updated to start Spark Connect on the Spark master container

Signed-off-by: Norman Jordan <norman.jordan@improving.com>
---
 docker/integ-test/.env                       |  1 +
 docker/integ-test/docker-compose.yml         |  5 +++++
 docker/integ-test/spark-master-entrypoint.sh | 17 +++++++++++++++++
 3 files changed, 23 insertions(+)
 create mode 100755 docker/integ-test/spark-master-entrypoint.sh

diff --git a/docker/integ-test/.env b/docker/integ-test/.env
index 421e854dc..cf73bdc89 100644
--- a/docker/integ-test/.env
+++ b/docker/integ-test/.env
@@ -4,6 +4,7 @@ DASHBOARDS_VERSION=latest
 MASTER_UI_PORT=8080
 MASTER_PORT=7077
 UI_PORT=4040
+SPARK_CONNECT_PORT=15002
 PPL_JAR=../../ppl-spark-integration/target/scala-2.12/ppl-spark-integration-assembly-0.7.0-SNAPSHOT.jar
 FLINT_JAR=../../flint-spark-integration/target/scala-2.12/flint-spark-integration-assembly-0.7.0-SNAPSHOT.jar
 OPENSEARCH_NODE_MEMORY=512m
diff --git a/docker/integ-test/docker-compose.yml b/docker/integ-test/docker-compose.yml
index f9e663d56..c5ee53d7d 100644
--- a/docker/integ-test/docker-compose.yml
+++ b/docker/integ-test/docker-compose.yml
@@ -6,6 +6,8 @@ services:
       - "${MASTER_UI_PORT:-8080}:8080"
       - "${MASTER_PORT:-7077}:7077"
       - "${UI_PORT:-4040}:4040"
+      - "${SPARK_CONNECT_PORT}:15002"
+    entrypoint: /opt/bitnami/scripts/spark/master-entrypoint.sh
     environment:
       - SPARK_MODE=master
       - SPARK_RPC_AUTHENTICATION_ENABLED=no
@@ -14,6 +16,9 @@ services:
       - SPARK_SSL_ENABLED=no
       - SPARK_PUBLIC_DNS=localhost
     volumes:
+      - type: bind
+        source: ./spark-master-entrypoint.sh
+        target: /opt/bitnami/scripts/spark/master-entrypoint.sh
       - type: bind
         source: ./spark-defaults.conf
         target: /opt/bitnami/spark/conf/spark-defaults.conf
diff --git a/docker/integ-test/spark-master-entrypoint.sh b/docker/integ-test/spark-master-entrypoint.sh
new file mode 100755
index 000000000..a21c20643
--- /dev/null
+++ b/docker/integ-test/spark-master-entrypoint.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+function start_spark_connect() {
+  sc_version=$(ls -1 /opt/bitnami/spark/jars/spark-core_*.jar | sed -e 's/^.*\/spark-core_//' -e 's/\.jar$//' -e 's/-/:/')
+
+  attempt=1
+  while [ -e "/tmp/spark_master_running" -a "$attempt" -le 10 ]; do
+    sleep 1
+    /opt/bitnami/spark/sbin/start-connect-server.sh --master spark://spark:7077 --packages org.apache.spark:spark-connect_${sc_version}
+    attempt=$(($attempt+1))
+  done
+}
+
+touch /tmp/spark_master_running
+start_spark_connect &
+/opt/bitnami/scripts/spark/entrypoint.sh /opt/bitnami/scripts/spark/run.sh
+rm /tmp/spark_master_running