kitchen sink postgres dependencies #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: [push] | |
| jobs: | |
| etl: | |
| runs-on: ubuntu-latest | |
| services: | |
| postgres: | |
| image: postgis/postgis:15-3.4-alpine | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| # Maps tcp port 5432 on service container to the host | |
| - 5432:5432 | |
| steps: | |
| - name: check out repo code | |
| uses: actions/checkout@v4 | |
| - name: Load Secrets | |
| uses: 1password/load-secrets-action@v1 | |
| with: | |
| export-env: true | |
| env: | |
| OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} | |
| DO_SPACES_ENDPOINT: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_ENDPOINT" | |
| DO_SPACES_ACCESS_KEY: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_ACCESS_KEY" | |
| DO_SPACES_SECRET_KEY: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_SECRET_KEY" | |
| DO_SPACES_BUCKET_DISTRIBUTIONS: "op://AE Data Flow/Digital Ocean - S3 file storage/DO_SPACES_BUCKET_DISTRIBUTIONS" | |
| DO_ZONING_API_DB_HOST: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API/host" | |
| DO_ZONING_API_DB_PORT: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API/port" | |
| DO_ZONING_API_DB_USERNAME_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/username" | |
| DO_ZONING_API_DB_PASSWORD_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/password" | |
| DO_ZONING_API_DB_DATABASE_DEV: "op://AE Data Flow/Digital Ocean DB Cluster - Zoning API dev/database" | |
| - name: Set .env file | |
| run: | | |
| echo "BUILD_ENGINE_HOST=127.0.0.1" >> .env | |
| echo "BUILD_ENGINE_PORT=5432" >> .env | |
| echo "BUILD_ENGINE_USER=postgres" >> .env | |
| echo "BUILD_ENGINE_PASSWORD=postgres" >> .env | |
| echo "BUILD_ENGINE_DB=postgres" >> .env | |
| echo "DO_SPACES_ENDPOINT=$DO_SPACES_ENDPOINT" >> .env | |
| echo "DO_SPACES_ACCESS_KEY=$DO_SPACES_ACCESS_KEY" >> .env | |
| echo "DO_SPACES_SECRET_KEY=$DO_SPACES_SECRET_KEY" >> .env | |
| echo "DO_SPACES_BUCKET_DISTRIBUTIONS=$DO_SPACES_BUCKET_DISTRIBUTIONS" >> .env | |
| echo "ZONING_API_HOST=$DO_ZONING_API_DB_HOST" >> .env | |
| echo "ZONING_API_PORT=$DO_ZONING_API_DB_PORT" >> .env | |
| echo "ZONING_API_USER=$DO_ZONING_API_DB_USERNAME_DEV" >> .env | |
| echo "ZONING_API_PASSWORD=$DO_ZONING_API_DB_PASSWORD_DEV" >> .env | |
| echo "ZONING_API_DB=$DO_ZONING_API_DB_DATABASE_DEV" >> .env | |
| - name: Install prerequisite packages | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y wget | |
| sudo apt-get install -y software-properties-common | |
| sudo apt install -y gpg gnupg2 apt-transport-https lsb-release ca-certificates | |
| - name: Remove postgresql 14 | |
| run: sudo apt-get remove -y postgresql-14 postgresql-client-14 | |
| - name: Install postgres-client-15 | |
| run: | | |
| sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' | |
| sudo wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - | |
| sudo apt-get update | |
| sudo apt-get install -y postgresql-15 postgresql-client-15 | |
| - name: Check postgres install | |
| run: pg_dump --version | |
| - name: Install minio client | |
| run: | | |
| sudo wget https://dl.min.io/client/mc/release/linux-amd64/mc | |
| sudo chmod +x mc | |
| sudo mv mc /usr/local/bin | |
| - name: Install python | |
| run: | | |
| sudo apt-get install -y python3 python3-pip | |
| pip install -r requirements.txt | |
| - name: Install dbt dependencies | |
| run: | | |
| sudo apt-get install -y git | |
| dbt deps | |
| - name: Download | |
| run: | | |
| ./bash/download.sh | |
| - name: Import | |
| run: | | |
| ./bash/import.sh | |
| - name: Transform | |
| run: | | |
| ./bash/transform.sh | |
| - name: Export | |
| run: | | |
| ./bash/export.sh | |