Update pr_tests_spark.yml

snowplow · Aug 6, 2024 · f9fb0b9 · f9fb0b9
1 parent 393d811
commit f9fb0b9
Showing 1 changed file with 7 additions and 1 deletion.
diff --git a/.github/workflows/pr_tests_spark.yml b/.github/workflows/pr_tests_spark.yml
@@ -90,6 +90,7 @@ jobs:
 
           ENV SPARK_VERSION=3.5.1
           ENV HADOOP_VERSION=3.3.4
+          ENV ICEBERG_VERSION=1.3.1
           ENV SPARK_HOME=/spark
 
           RUN apt-get update && apt-get install -y curl wget procps rsync ssh iputils-ping net-tools jq
@@ -108,7 +109,7 @@ jobs:
           # Install AWS Glue libraries and Iceberg
           RUN curl -L -o /spark/jars/aws-java-sdk-bundle-1.11.1026.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.11.1026/aws-java-sdk-bundle-1.11.1026.jar && \
               curl -L -o /spark/jars/hadoop-aws-3.3.4.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar && \
-              curl -L -o /spark/jars/iceberg-spark-runtime-3.5_2.12-1.3.1.jar https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/1.3.1/iceberg-spark-runtime-3.5_2.12-1.3.1.jar
+              curl -L -o /spark/jars/iceberg-spark-runtime-3.5_2.12-\${ICEBERG_VERSION}.jar https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/\${ICEBERG_VERSION}/iceberg-spark-runtime-3.5_2.12-\${ICEBERG_VERSION}.jar
 
           WORKDIR \${SPARK_HOME}
 
@@ -246,6 +247,11 @@ jobs:
           echo "Verifying Spark configuration..."
           docker-compose exec -T spark-master bash -c "cat /spark/conf/spark-defaults.conf"
 
+      - name: List Spark jars
+        run: |
+          echo "Listing Spark jars..."
+          docker-compose exec -T spark-master bash -c "ls -l /spark/jars | grep iceberg"
+
       - name: Wait for Thrift Server
         run: |
           echo "Waiting for Thrift Server to be fully operational..."