Fixes for host

snowplow · Aug 2, 2024 · a261a7c · a261a7c
1 parent 4ba1d9d
commit a261a7c
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 5 deletions.
diff --git a/.github/workflows/pr_tests_spark.yml b/.github/workflows/pr_tests_spark.yml
@@ -120,7 +120,7 @@ jobs:
                 - "10000:10000"
               environment:
                 - SPARK_MODE=master
-                - SPARK_MASTER_HOST=spark-master
+                - SPARK_MASTER_HOST=localhost
                 - SPARK_MASTER_PORT=7077
                 - SPARK_MASTER_WEBUI_PORT=8080
 
@@ -163,11 +163,15 @@ jobs:
         run: |
           echo "Waiting for Thrift Server to be fully operational..."
           sleep 30
-
+      - name: Check Thrift Server status
+        run: |
+          docker-compose exec -T spark-master bash -c "curl localhost:10000"
+      - name: DBT debug
+        run: |
+          dbt debug --target spark
       - name: 'Pre-test: Drop ci schemas'
         run: |
           dbt run-operation post_ci_cleanup --target spark
-
       - name: Run tests
         run: ./.scripts/integration_tests.sh -d spark
 

diff --git a/.github/workflows/pr_tests_spark_simple.yml b/.github/workflows/pr_tests_spark_simple.yml
@@ -70,7 +70,7 @@ jobs:
               - "7077:7077"
             environment:
               - SPARK_MODE=master
-              - SPARK_MASTER_HOST=spark-master
+              - SPARK_MASTER_HOST=localhost
               - SPARK_MASTER_PORT=7077
               - SPARK_MASTER_WEBUI_PORT=8080
 
@@ -129,6 +129,7 @@ jobs:
       run: |
         echo "Running Spark Pi example job..."
         docker-compose exec -T spark-master bin/spark-submit --master spark://spark-master:7077 --class org.apache.spark.examples.SparkPi examples/jars/spark-examples_2.12-3.5.1.jar 10
+        
     - name: 'Pre-test: Drop ci schemas'
       run: dbt run-operation post_ci_cleanup --target spark
 

diff --git a/integration_tests/ci/profiles.yml b/integration_tests/ci/profiles.yml
@@ -71,7 +71,7 @@ integration_tests:
     spark:
       type: spark
       method: thrift
-      host: "{{ env_var('SPARK_MASTER_HOST') }}"
+      host: "{{ env_var('SPARK_MASTER_HOST', 'localhost') }}"
       port: 10000
       user: "{{ env_var('SPARK_USER', 'spark') }}"
       schema: "{{ env_var('SPARK_SCHEMA', 'default') }}"