Update pr_tests_spark.yml

snowplow · Aug 6, 2024 · 9e520f3 · 9e520f3
1 parent daa9695
commit 9e520f3
Showing 1 changed file with 25 additions and 15 deletions.
diff --git a/.github/workflows/pr_tests_spark.yml b/.github/workflows/pr_tests_spark.yml
@@ -112,15 +112,15 @@ jobs:
               command: |
                 bash -c "
                 /spark/sbin/start-master.sh &&
-                /spark/sbin/start-thriftserver.sh --master spark://spark-master:7077 &&
+                /spark/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 --name 'Thrift JDBC/ODBC Server' spark-internal &&
                 tail -f /spark/logs/*"
               ports:
                 - "8080:8080"
                 - "7077:7077"
                 - "10000:10000"
               environment:
                 - SPARK_MODE=master
-                - SPARK_MASTER_HOST=localhost
+                - SPARK_MASTER_HOST=0.0.0.0
                 - SPARK_MASTER_PORT=7077
                 - SPARK_MASTER_WEBUI_PORT=8080
 
@@ -152,7 +152,12 @@ jobs:
       - name: Wait for services to start
         run: |
           echo "Waiting for Spark services to start..."
-          sleep 60
+          sleep 120
+
+      - name: Check Docker network
+        run: |
+          docker network ls
+          docker network inspect integration_tests_default
 
       - name: Print Docker logs
         run: |
@@ -173,21 +178,26 @@ jobs:
           docker-compose exec -T spark-master bash -c "jps && ps aux | grep spark && netstat -tuln"
           docker-compose exec -T spark-worker bash -c "jps && ps aux | grep spark && netstat -tuln"
 
-      - name: Check Logs Spark Master
-        run: docker-compose logs spark-master
+      - name: Debug Spark Master
+        run: |
+          docker-compose exec -T spark-master bash -c "cat /spark/logs/spark--org.apache.spark.deploy.master.Master-*.out"
+
+      - name: Debug ThriftServer
+        run: |
+          docker-compose exec -T spark-master bash -c "cat /spark/logs/spark--org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-*.out"
+
+      - name: Verify Spark configuration
+        run: |
+          docker-compose exec -T spark-master bash -c "cat /spark/conf/spark-defaults.conf"
+
       - name: Check if the Thrift Server is running in Spark Master
-        run: docker-compose exec spark-master bash -c "ps aux | grep ThriftServer"
+        run: docker-compose exec -T spark-master bash -c "ps aux | grep ThriftServer"
+
       - name: Check if port 10000 is actually listening inside the spark-master container
-        run: docker-compose exec spark-master bash -c "netstat -tuln | grep 10000"
-      - name: Try to connect to the Thrift server from the spark-master container itself
-        run: docker-compose exec spark-master bash -c "curl localhost:10000"
-      - name: Verify Spark configuration
-        run: docker-compose exec spark-master bash -c "cat /spark/conf/spark-defaults.conf"
+        run: docker-compose exec -T spark-master bash -c "netstat -tuln | grep 10000"
 
-      - name: Check Thrift Server
-        run: |
-          docker-compose exec -T spark-master bash -c "curl localhost:10000"
-          docker-compose exec -T spark-master bash -c "netstat -tuln | grep 10000"
+      - name: Try to connect to the Thrift server from the spark-master container itself
+        run: docker-compose exec -T spark-master bash -c "curl -v localhost:10000"
 
       - name: Wait for Thrift Server
         run: |