Image again

snowplow · Aug 8, 2024 · 41075a1 · 41075a1
1 parent 641eba5
commit 41075a1
Show file tree

Hide file tree

Showing 3 changed files with 28 additions and 5 deletions.
diff --git a/.github/workflows/pr_tests_spark.yml b/.github/workflows/pr_tests_spark.yml
@@ -29,6 +29,11 @@ jobs:
     steps:
       - name: Check out
         uses: actions/checkout@v3
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set SCHEMA_SUFFIX env
         run: >-
@@ -66,11 +71,9 @@ jobs:
         run: |
           sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
           sudo chmod +x /usr/local/bin/docker-compose
-
       - name: Build and start Spark cluster
         working-directory: .github/workflows/spark_deployment
         run: |
-          docker-compose build
           docker-compose up -d
           echo "Waiting for Spark services to start..."
           sleep 90

diff --git a/.github/workflows/spark_deployment/build_and_push.sh b/.github/workflows/spark_deployment/build_and_push.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Set variables
+DOCKER_HUB_ORG="snowplow"
+IMAGE_NAME="spark-s3-iceberg"
+TAG="latest"
+
+# Build the image
+echo "Building Docker image..."
+docker build --platform linux/amd64 -t $DOCKER_HUB_ORG/$IMAGE_NAME:$TAG .
+
+# Log in to Docker Hub
+echo "Logging in to Docker Hub..."
+docker login
+
+# Push the image to Docker Hub
+echo "Pushing image to Docker Hub..."
+docker push $DOCKER_HUB_ORG/$IMAGE_NAME:$TAG
+
+echo "Image successfully built and pushed to Docker Hub"
diff --git a/.github/workflows/spark_deployment/docker-compose.yml b/.github/workflows/spark_deployment/docker-compose.yml
@@ -6,7 +6,7 @@ networks:
 
 services:
   spark-master:
-    build: .
+    image: snowplow/spark-s3-iceberg:latest
     command: ["/bin/bash", "-c", "/spark/sbin/start-master.sh -h spark-master --properties-file /spark/conf/spark-defaults.conf && tail -f /spark/logs/spark--org.apache.spark.deploy.master.Master-1-*.out"]
     hostname: spark-master
     ports:
@@ -27,7 +27,7 @@ services:
       - spark-network
 
   spark-worker:
-    build: .
+    image: snowplow/spark-s3-iceberg:latest
     command: ["/bin/bash", "-c", "sleep 10 && /spark/sbin/start-worker.sh spark://spark-master:7077 --properties-file /spark/conf/spark-defaults.conf && tail -f /spark/logs/spark--org.apache.spark.deploy.worker.Worker-*.out"]
     depends_on:
       - spark-master
@@ -47,7 +47,7 @@ services:
       - spark-network
 
   thrift-server:
-    build: .
+    image: snowplow/spark-s3-iceberg:latest
     command: ["/bin/bash", "-c", "sleep 30 && /spark/sbin/start-thriftserver.sh --master spark://spark-master:7077 --driver-memory 2g --executor-memory 3g --hiveconf hive.server2.thrift.port=10000 --hiveconf hive.server2.thrift.bind.host=0.0.0.0 --conf spark.sql.hive.thriftServer.async=true --conf spark.sql.hive.thriftServer.workerQueue.size=2000 --conf spark.sql.hive.thriftServer.maxWorkerThreads=100 --conf spark.sql.hive.thriftServer.minWorkerThreads=50 && tail -f /spark/logs/spark--org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-*.out"]
     ports:
       - '10000:10000'