Update pr_tests_spark.yml

snowplow · Aug 6, 2024 · db55258 · db55258
1 parent 2f1869a
commit db55258
Showing 1 changed file with 15 additions and 19 deletions.
diff --git a/.github/workflows/pr_tests_spark.yml b/.github/workflows/pr_tests_spark.yml
@@ -96,6 +96,10 @@ jobs:
 
           ENV PATH=\$PATH:\${SPARK_HOME}/bin:\${SPARK_HOME}/sbin
 
+          RUN mkdir -p /spark/spark-warehouse && \
+              chown -R root:root /spark/spark-warehouse && \
+              chmod -R 777 /spark/spark-warehouse
+
           WORKDIR \${SPARK_HOME}
 
           CMD ["bash"]
@@ -124,9 +128,6 @@ jobs:
               build: .
               command: |
                 bash -c "
-                mkdir -p /spark/spark-warehouse &&
-                chown -R root:root /spark/spark-warehouse &&
-                chmod -R 777 /spark/spark-warehouse &&
                 /spark/sbin/start-master.sh &&
                 /spark/sbin/start-thriftserver.sh --master spark://spark-master:7077 &&
                 tail -f /spark/logs/*"
@@ -142,6 +143,7 @@ jobs:
                 - SPARK_MASTER_WEBUI_PORT=8080
               volumes:
                 - ./spark-defaults.conf:/spark/conf/spark-defaults.conf
+                - spark-warehouse:/spark/spark-warehouse
               networks:
                 - spark-network
 
@@ -157,25 +159,27 @@ jobs:
                 - SPARK_WORKER_PORT=8081
                 - SPARK_WORKER_WEBUI_PORT=8081
                 - SPARK_MASTER=spark://spark-master:7077
+              volumes:
+                - spark-warehouse:/spark/spark-warehouse
               networks:
                 - spark-network
+
+          volumes:
+            spark-warehouse:
           EOF
 
       - name: Debug Docker Compose file
         run: cat docker-compose.yml
 
-      - name: Set Spark user permissions
-        run: |
-          docker-compose exec -T spark-master bash -c "
-          mkdir -p /spark/spark-warehouse &&
-          chown -R root:root /spark/spark-warehouse &&
-          chmod -R 777 /spark/spark-warehouse
-          "
-
       - name: Build and start Spark cluster
         run: |
           docker-compose build --no-cache
           docker-compose up -d
+          echo "Waiting for Spark services to start..."
+          sleep 60
+
+      - name: Check running containers
+        run: docker ps
 
       - name: Initialize Hive Metastore
         run: |
@@ -196,14 +200,6 @@ jobs:
           ls -la /spark
           "
 
-      - name: Check running containers
-        run: docker ps
-
-      - name: Wait for services to start
-        run: |
-          echo "Waiting for Spark services to start..."
-          sleep 120
-
       - name: Check Docker network
         run: |
           docker network ls