test_update_api_database.yml

on: [push]

jobs:
    etl:
        runs-on: ubuntu-latest
        services:
            postgres:
                image: postgis/postgis:15-3.4-alpine
                env:
                    POSTGRES_PASSWORD: postgres
                options: >-
                    --health-cmd pg_isready
                    --health-interval 10s
                    --health-timeout 5s
                    --health-retries 5
                ports:
                    # Maps tcp port 5432 on service container to the host
                    - 5432:5432
        steps:
            - name: check out repo code
              uses: actions/checkout@v4
            - name: Load Secrets
              uses: 1password/load-secrets-action@v1  
              with:
                    export-env: true
              env:
                    OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
                    DO_SPACES_ENDPOINT: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_ENDPOINT"
                    DO_SPACES_ACCESS_KEY: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_ACCESS_KEY"
                    DO_SPACES_SECRET_KEY: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_SECRET_KEY"
                    DO_SPACES_BUCKET_DISTRIBUTIONS: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_BUCKET_DISTRIBUTIONS"
                    DO_ZONING_API_DB_HOST: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API/host"
                    DO_ZONING_API_DB_PORT: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API/port"
                    DO_ZONING_API_DB_USERNAME_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/username"
                    DO_ZONING_API_DB_PASSWORD_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/password"
                    DO_ZONING_API_DB_DATABASE_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/database"
            - name: Set .env file
              run: |
                echo "BUILD_ENGINE_HOST=127.0.0.1" >> .env
                echo "BUILD_ENGINE_PORT=5432" >> .env
                echo "BUILD_ENGINE_USER=postgres" >> .env
                echo "BUILD_ENGINE_PASSWORD=postgres" >> .env
                echo "BUILD_ENGINE_DB=postgres" >> .env
                echo "DO_SPACES_ENDPOINT=$DO_SPACES_ENDPOINT" >> .env
                echo "DO_SPACES_ACCESS_KEY=$DO_SPACES_ACCESS_KEY" >> .env
                echo "DO_SPACES_SECRET_KEY=$DO_SPACES_SECRET_KEY" >> .env
                echo "DO_SPACES_BUCKET_DISTRIBUTIONS=$DO_SPACES_BUCKET_DISTRIBUTIONS" >> .env
                echo "ZONING_API_HOST=$DO_ZONING_API_DB_HOST" >> .env
                echo "ZONING_API_PORT=$DO_ZONING_API_DB_PORT" >> .env
                echo "ZONING_API_USER=$DO_ZONING_API_DB_USERNAME_DEV" >> .env
                echo "ZONING_API_PASSWORD=$DO_ZONING_API_DB_PASSWORD_DEV" >> .env
                echo "ZONING_API_DB=$DO_ZONING_API_DB_DATABASE_DEV" >> .env

            - name: Install prerequisite packages
              run: |
                sudo apt-get update  
                sudo apt-get install -y wget
                sudo apt-get install -y software-properties-common

            - name: Remove postgresql 14
              run: sudo apt-get remove -y postgresql-14 postgresql-client-14

            - name: Install postgres-client-15
              run: |                
                sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
                sudo wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
                sudo apt-get update
                sudo apt-get install -y postgresql-client-15

            - name: Check postgres install
              run: pg_dump --version    

            - name: Install minio client
              run: |
                sudo wget https://dl.min.io/client/mc/release/linux-amd64/mc
                sudo chmod +x mc
                sudo mv mc /usr/local/bin
            - name: Install python
              run: |
                sudo apt-get install -y python3 python3-pip
                pip install -r requirements.txt 

            - name: Install dbt dependencies
              run: |
                sudo apt-get install -y git
                dbt deps

            - name: Download
              run: |
                ./bash/download.sh

            - name: Import
              run: |
                ./bash/import.sh

            - name: Transform
              run: |
                ./bash/transform.sh

            - name: Export
              run: |
                ./bash/export.sh