Prepare Utils codebase for Spark ( Iceberg ) Support #41
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: pr_tests_spark | |
| on: | |
| pull_request: | |
| env: | |
| DBT_PROFILES_DIR: ./ci | |
| SPARK_MASTER_HOST: localhost | |
| SPARK_USER: spark | |
| SPARK_SCHEMA: default | |
| AWS_REGION: eu-west-1 | |
| AWS_DEFAULT_REGION: eu-west-1 | |
| DOCKER_PLATFORM: linux/amd64 | |
| jobs: | |
| pr_tests_spark: | |
| name: pr_tests_spark | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: ./integration_tests | |
| strategy: | |
| matrix: | |
| dbt_version: | |
| - 1.* | |
| warehouse: | |
| - spark | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v3 | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Set SCHEMA_SUFFIX env | |
| run: >- | |
| echo "SCHEMA_SUFFIX=$(echo ${DBT_VERSION%.*} | tr . _)" >> $GITHUB_ENV | |
| env: | |
| DBT_VERSION: '${{ matrix.dbt_version }}' | |
| - name: Set DEFAULT_TARGET env | |
| run: | | |
| echo "DEFAULT_TARGET=${{ matrix.warehouse }}" >> $GITHUB_ENV | |
| - name: Python setup | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.8.x | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip wheel setuptools | |
| pip install -Iv "dbt-spark[PyHive]==${{ matrix.dbt_version }}" --upgrade | |
| pip install boto3 awscli | |
| dbt deps | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: eu-west-1 | |
| - name: Install Docker Compose | |
| run: | | |
| sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - name: Pull Docker image | |
| run: docker pull snowplow/spark-s3-iceberg:v2 | |
| - name: Check image architecture | |
| run: docker inspect --format='{{.Architecture}}' snowplow/spark-s3-iceberg:v2 | |
| - name: Diagnostic Information | |
| run: | | |
| echo "Docker version:" | |
| docker --version | |
| echo "Docker info:" | |
| docker info | |
| echo "Available images:" | |
| docker images | |
| echo "System information:" | |
| uname -a | |
| - name: Start Spark cluster | |
| working-directory: .github/workflows/spark_deployment | |
| run: | | |
| docker-compose config | |
| docker-compose pull | |
| docker-compose up -d | |
| echo "Waiting for Spark services to start..." | |
| sleep 30 | |
| docker-compose ps | |
| docker-compose logs | |
| - name: Check running containers | |
| working-directory: .github/workflows/spark_deployment | |
| run: | | |
| docker ps -a | |
| echo "Container details:" | |
| docker inspect $(docker ps -aq) | |
| - name: Print Docker logs | |
| if: failure() | |
| working-directory: .github/workflows/spark_deployment | |
| run: | | |
| echo "Docker logs for spark-master:" | |
| docker-compose logs spark-master | |
| echo "Docker logs for spark-worker:" | |
| docker-compose logs spark-worker | |
| echo "Docker logs for thrift-server:" | |
| docker-compose logs thrift-server | |
| - name: "Pre-test: Drop ci schemas" | |
| run: | | |
| dbt run-operation post_ci_cleanup --target spark | |
| - name: Run tests | |
| run: ./.scripts/integration_tests.sh -d spark | |
| - name: "Post-test: Drop ci schemas" | |
| run: | | |
| dbt run-operation post_ci_cleanup --target spark |