ci.yaml

name: CI

on:
  workflow_dispatch:
  push:
    paths:
      - 'columnar/**'
      - 'third-party/**'
      - 'files/**'
      - 'acceptance/**'
      - docker-bake.hcl
      - Dockerfile
      - Makefile
      - VERSION
      - HYDRA_PROD_VER
      - .github/workflows/ci.yaml

env:
  POSTGRES_REPO: ghcr.io/hydradatabase/hydra
  SPILO_REPO: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo
  TEST_ARTIFACT_DIR: /tmp/test_artifacts

jobs:
  lint_go:
    name: Lint acceptance
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - uses: actions/setup-go@v3
        with:
          go-version-file: acceptance/go.mod
          cache-dependency-path: acceptance/go.sum
          check-latest: true
          cache: true
      - name: Lint
        run: make lint_acceptance

  validate_columnar:
    name: Validate Columnar ${{ matrix.postgres }}
    strategy:
      fail-fast: false
      matrix:
        postgres: ["13", "14", "15", "16"]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Depot
        uses: depot/setup-action@v1
        with:
          version: latest
      - name: Bake Columnar validation
        uses: depot/bake-action@v1
        with:
          token: ${{ secrets.DEPOT_TOKEN }}
          project: ${{ secrets.DEPOT_PROJECT }}
          pull: true
          targets: ${{ format('columnar_{0}', matrix.postgres) }}
          set: |
            *.platform=linux/amd64
            columnar.target=checker

  build_validate_postgres:
    name: Build and Validate Postgres ${{ matrix.postgres }}
    needs: [validate_columnar]
    strategy:
      fail-fast: false
      matrix:
        postgres: ["13", "14", "15", "16"]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: setup POSTGRES_IMAGE env
        run: echo "POSTGRES_IMAGE=${{ format('{0}:{1}-{2}', env.POSTGRES_REPO, matrix.postgres, github.sha) }}" >> $GITHUB_ENV
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Depot
        uses: depot/setup-action@v1
      - name: Bake and load amd64 Postgres image
        uses: depot/bake-action@v1
        env:
          POSTGRES_BASE_VERSION: ${{ matrix.postgres }}
        with:
          token: ${{ secrets.DEPOT_TOKEN }}
          project: ${{ secrets.DEPOT_PROJECT }}
          load: true
          pull: true
          targets: postgres
          set: |
            *.platform=linux/amd64
            postgres.tags=${{ env.POSTGRES_IMAGE }}
      - uses: actions/setup-go@v3
        with:
          go-version-file: acceptance/go.mod
          cache-dependency-path: acceptance/go.sum
          cache: true
      - name: Run Acceptance Tests
        env:
          POSTGRES_UPGRADE_FROM_IMAGE: ${{ format('{0}:{1}', env.POSTGRES_REPO, matrix.postgres) }}
          POSTGRES_BASE_VERSION: ${{ matrix.postgres }}
          TEST_AWS_ACCESS_KEY_ID: ${{ secrets.TEST_FIXTURE_AWS_ACCESS_KEY_ID }}
          TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_FIXTURE_AWS_SECRET_ACCESS_KEY }}
          TEST_AWS_REGION: us-east-1
          TEST_AWS_S3_BUCKET: hydra-test-fixtures
        run: make postgres_pull_upgrade_image postgres_acceptance_test
      - name: tgz container logs
        if: failure()
        run: |
          tar -czf /tmp/test_results.tar.gz ${{ env.TEST_ARTIFACT_DIR }}
      - name: Upload test container logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: postgres_${{ matrix.postgres }}_container_logs
          path: /tmp/test_results.tar.gz
          retention-days: 7
      - name: Bake multi-platform Postgres
        if: github.ref == 'refs/heads/main'
        uses: depot/bake-action@v1
        with:
          token: ${{ secrets.DEPOT_TOKEN }}
          project: ${{ secrets.DEPOT_PROJECT }}
          pull: true
          targets: postgres

  build_validate_spilo:
    name: Build and Validate Spilo
    needs: [validate_columnar]
    if: github.repository == 'hydradatabase/hydra-internal'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: setup SPILO_IMAGE
        run: echo "SPILO_IMAGE=${{ format('{0}:{1}', env.SPILO_REPO, github.sha) }}" >> $GITHUB_ENV
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Depot
        uses: depot/setup-action@v1
      - name: Bake and load amd64 spilo image
        uses: depot/bake-action@v1
        with:
          token: ${{ secrets.DEPOT_TOKEN }}
          project: ${{ secrets.DEPOT_PROJECT }}
          load: true
          pull: true
          targets: spilo
          set: |
            *.platform=linux/amd64
            spilo.tags=${{ env.SPILO_IMAGE }}
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
          aws-region: us-east-1
          mask-aws-account-id: no
      - name: Login to Amazon ECR
        id: spilo-ecr
        uses: aws-actions/amazon-ecr-login@v1
        with:
          registries: "011789831835"
      - uses: actions/setup-go@v3
        with:
          go-version-file: acceptance/go.mod
          cache-dependency-path: acceptance/go.sum
          cache: true
      - name: Run Acceptance Tests for Postgres 13
        env:
          SPILO_POSTGRES_VERSION: "13"
          TEST_AWS_ACCESS_KEY_ID: ${{ secrets.TEST_FIXTURE_AWS_ACCESS_KEY_ID }}
          TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_FIXTURE_AWS_SECRET_ACCESS_KEY }}
          TEST_AWS_REGION: us-east-1
          TEST_AWS_S3_BUCKET: hydra-test-fixtures
        run: make spilo_pull_upgrade_image spilo_acceptance_test
      - name: Run Acceptance Tests for Postgres 14
        env:
          SPILO_POSTGRES_VERSION: "14"
          TEST_AWS_ACCESS_KEY_ID: ${{ secrets.TEST_FIXTURE_AWS_ACCESS_KEY_ID }}
          TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_FIXTURE_AWS_SECRET_ACCESS_KEY }}
          TEST_AWS_REGION: us-east-1
          TEST_AWS_S3_BUCKET: hydra-test-fixtures
        run: make spilo_acceptance_test
      - name: tgz container logs
        if: failure()
        run: |
          tar -czf /tmp/test_results.tar.gz ${{ env.TEST_ARTIFACT_DIR }}
      - name: Upload test container logs
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: spilo_container_logs
          path: /tmp/test_results.tar.gz
          retention-days: 7
      - name: Bake amd64 spilo
        if: github.ref == 'refs/heads/main'
        uses: depot/bake-action@v1
        with:
          token: ${{ secrets.DEPOT_TOKEN }}
          project: ${{ secrets.DEPOT_PROJECT }}
          pull: true
          targets: spilo
          set: |
            *.platform=linux/amd64

  push_postgres:
    name: Push Postgres ${{ matrix.postgres }}
    needs: [build_validate_postgres]
    if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main'
    strategy:
      fail-fast: false
      matrix:
        postgres: ["13", "14", "15", "16"]
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Depot
        uses: depot/setup-action@v1
      - name: Login to ghcr.io
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Bake and push Postgres
        uses: depot/bake-action@v1
        env:
          POSTGRES_BASE_VERSION: ${{ matrix.postgres }}
        with:
          token: ${{ secrets.DEPOT_TOKEN }}
          project: ${{ secrets.DEPOT_PROJECT }}
          push: true
          targets: postgres
          set: |
            postgres.tags=${{ env.POSTGRES_REPO }}:${{ matrix.postgres }}
            postgres.tags=${{ env.POSTGRES_REPO }}:${{ matrix.postgres }}-${{ github.sha }}
            ${{ matrix.postgres == '14' && format('postgres.tags={0}:latest', env.POSTGRES_REPO) || '' }}

  push_spilo:
    name: Push Spilo
    needs: [build_validate_spilo]
    if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main'
    uses: ./.github/workflows/build_spilo.yaml
    with:
      production: true
      spilo_repo: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo # env vars can't be used here
    secrets: inherit

  benchmark_postgres:
    name: Benchmark Hydra Postgres image (${{ matrix.benchmark }})
    needs: [push_postgres]
    if: github.repository == 'hydradatabase/hydra' && github.ref == 'refs/heads/main'
    strategy:
      fail-fast: false
      matrix:
        benchmark: ["clickbench-100M", "warehouse-10G"]
    uses: ./.github/workflows/benchmark.yaml
    with:
      benchmark: ${{ matrix.benchmark }}
      repo: ghcr.io/hydradatabase/hydra
      tag: 14-${{ github.sha }}
    secrets: inherit

  benchmark_spilo:
    name: Benchmark Hydra Spilo image (${{ matrix.benchmark }})
    needs: [push_spilo]
    if: github.repository == 'hydradatabase/hydra-internal' && github.ref == 'refs/heads/main'
    strategy:
      fail-fast: false
      matrix:
        benchmark: ["clickbench-100M", "warehouse-10G"]
    uses: ./.github/workflows/benchmark.yaml
    with:
      benchmark: ${{ matrix.benchmark }}
      repo: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo
      tag: ${{ github.sha }}
    secrets: inherit