test_update_api_database.yml

on: [push]

jobs:
    etl:
        runs-on: ubuntu-latest
        services:
            postgres:
                image: postgis/postgis:15-3.4-alpine
                env:
                    POSTGRES_PASSWORD: postgres
                options: >-
                    --health-cmd pg_isready
                    --health-interval 10s
                    --health-timeout 5s
                    --health-retries 5
                ports:
                    # Maps tcp port 5432 on service container to the host
                    - 5432:5432
        steps:
            - name: check out repo code
              uses: actions/checkout@v4
            - name: Load Secrets
              uses: 1password/load-secrets-action@v1  
              with:
                    export-env: true
              env:
                    OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
                    DO_SPACES_ENDPOINT: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_ENDPOINT"
                    DO_SPACES_ACCESS_KEY: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_ACCESS_KEY"
                    DO_SPACES_SECRET_KEY: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_SECRET_KEY"
                    DO_SPACES_BUCKET_DISTRIBUTIONS: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_BUCKET_DISTRIBUTIONS"
                    DO_ZONING_API_DB_HOST: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API/host"
                    DO_ZONING_API_DB_PORT: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API/port"
                    DO_ZONING_API_DB_USERNAME_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/username"
                    DO_ZONING_API_DB_PASSWORD_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/password"
                    DO_ZONING_API_DB_DATABASE_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/database"
            - name: Set .env file
              run: |
                echo "BUILD_ENGINE_HOST=127.0.0.1" >> .env
                echo "BUILD_ENGINE_PORT=5432" >> .env
                echo "BUILD_ENGINE_USER=postgres" >> .env
                echo "BUILD_ENGINE_PASSWORD=postgres" >> .env
                echo "BUILD_ENGINE_DB=postgres" >> .env
                echo "DO_SPACES_ENDPOINT=$DO_SPACES_ENDPOINT" >> .env
                echo "DO_SPACES_ACCESS_KEY=$DO_SPACES_ACCESS_KEY" >> .env
                echo "DO_SPACES_SECRET_KEY=$DO_SPACES_SECRET_KEY" >> .env
                echo "DO_SPACES_BUCKET_DISTRIBUTIONS=$DO_SPACES_BUCKET_DISTRIBUTIONS" >> .env
                echo "ZONING_API_HOST=$DO_ZONING_API_DB_HOST" >> .env
                echo "ZONING_API_PORT=$DO_ZONING_API_DB_PORT" >> .env
                echo "ZONING_API_USER=$DO_ZONING_API_DB_USERNAME_DEV" >> .env
                echo "ZONING_API_PASSWORD=$DO_ZONING_API_DB_PASSWORD_DEV" >> .env
                echo "ZONING_API_DB=$DO_ZONING_API_DB_DATABASE_DEV" >> .env

            - name: Install prerequisite packages
              run: |
                sudo apt-get update  
                sudo apt-get install -y wget
                sudo apt-get install -y git

            - name: Setup PostgreSQL
              uses: tj-actions/install-postgresql@v3
              with:
                postgresql-version: 15

            - name: Check postgres install
              run: pg_dump --version    

            - name: Install minio client
              run: |
                sudo wget https://dl.min.io/client/mc/release/linux-amd64/mc
                sudo chmod +x mc
                sudo mv mc /usr/local/bin

            - name: Setup python
              uses: actions/setup-python@v5
              with:
                python-version-file: ".python-version"
          
            - name: Install python dependencies
              run: pip install -r requirements.txt 

            - name: Install dbt dependencies
              run: dbt deps

            - name: Download
              run: ./bash/download.sh

            - name: Import
              run: ./bash/import.sh

            - name: Transform
              run: ./bash/transform.sh

            - name: Export
              run: ./bash/export.sh