From 9e520f38468b1a1c5ee93eaeac34d0b358d45403 Mon Sep 17 00:00:00 2001
From: Ilias Xenogiannis <ilias1111@users.noreply.github.com>
Date: Tue, 6 Aug 2024 12:47:07 +0300
Subject: [PATCH] Update pr_tests_spark.yml

---
 .github/workflows/pr_tests_spark.yml | 40 +++++++++++++++++-----------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/pr_tests_spark.yml b/.github/workflows/pr_tests_spark.yml
index eee16842..a5071409 100644
--- a/.github/workflows/pr_tests_spark.yml
+++ b/.github/workflows/pr_tests_spark.yml
@@ -112,7 +112,7 @@ jobs:
               command: |
                 bash -c "
                 /spark/sbin/start-master.sh &&
-                /spark/sbin/start-thriftserver.sh --master spark://spark-master:7077 &&
+                /spark/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 --name 'Thrift JDBC/ODBC Server' spark-internal &&
                 tail -f /spark/logs/*"
               ports:
                 - "8080:8080"
@@ -120,7 +120,7 @@ jobs:
                 - "10000:10000"
               environment:
                 - SPARK_MODE=master
-                - SPARK_MASTER_HOST=localhost
+                - SPARK_MASTER_HOST=0.0.0.0
                 - SPARK_MASTER_PORT=7077
                 - SPARK_MASTER_WEBUI_PORT=8080
 
@@ -152,7 +152,12 @@ jobs:
       - name: Wait for services to start
         run: |
           echo "Waiting for Spark services to start..."
-          sleep 60
+          sleep 120
+
+      - name: Check Docker network
+        run: |
+          docker network ls
+          docker network inspect integration_tests_default
 
       - name: Print Docker logs
         run: |
@@ -173,21 +178,26 @@ jobs:
           docker-compose exec -T spark-master bash -c "jps && ps aux | grep spark && netstat -tuln"
           docker-compose exec -T spark-worker bash -c "jps && ps aux | grep spark && netstat -tuln"
 
-      - name: Check Logs Spark Master
-        run: docker-compose logs spark-master
+      - name: Debug Spark Master
+        run: |
+          docker-compose exec -T spark-master bash -c "cat /spark/logs/spark--org.apache.spark.deploy.master.Master-*.out"
+
+      - name: Debug ThriftServer
+        run: |
+          docker-compose exec -T spark-master bash -c "cat /spark/logs/spark--org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-*.out"
+
+      - name: Verify Spark configuration
+        run: |
+          docker-compose exec -T spark-master bash -c "cat /spark/conf/spark-defaults.conf"
+
       - name: Check if the Thrift Server is running in Spark Master
-        run: docker-compose exec spark-master bash -c "ps aux | grep ThriftServer"
+        run: docker-compose exec -T spark-master bash -c "ps aux | grep ThriftServer"
+
       - name: Check if port 10000 is actually listening inside the spark-master container
-        run: docker-compose exec spark-master bash -c "netstat -tuln | grep 10000"
-      - name: Try to connect to the Thrift server from the spark-master container itself
-        run: docker-compose exec spark-master bash -c "curl localhost:10000"
-      - name: Verify Spark configuration
-        run: docker-compose exec spark-master bash -c "cat /spark/conf/spark-defaults.conf"
+        run: docker-compose exec -T spark-master bash -c "netstat -tuln | grep 10000"
 
-      - name: Check Thrift Server
-        run: |
-          docker-compose exec -T spark-master bash -c "curl localhost:10000"
-          docker-compose exec -T spark-master bash -c "netstat -tuln | grep 10000"
+      - name: Try to connect to the Thrift server from the spark-master container itself
+        run: docker-compose exec -T spark-master bash -c "curl -v localhost:10000"
 
       - name: Wait for Thrift Server
         run: |