fix: Fix dependencies and the login #669
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: EpiGraphHub | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| env: | |
| _AIRFLOW_DB_UPGRADE: true | |
| _AIRFLOW_WWW_USER_CREATE: true | |
| _AIRFLOW_WWW_USER_EMAIL: [email protected] | |
| _AIRFLOW_WWW_USER_FIRST_NAME: Super | |
| _AIRFLOW_WWW_USER_LAST_NAME: Admin | |
| _AIRFLOW_WWW_USER_PASSWORD: airflow | |
| _AIRFLOW_WWW_USER_USERNAME: airflow | |
| AIRFLOW_FILES_PATH_DIR_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/containers/airflow | |
| AIRFLOW_HOME: /opt/airflow | |
| AIRFLOW_PORT: 8099 | |
| AIRFLOW__API__AUTH_BACKENDS: airflow.api.auth.backend.basic_auth | |
| AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: true | |
| AIRFLOW__CORE__FERNET_KEY: rhZaG4rip4DrA4Z93rI9uqWVK2/8Tk+zSixmEe3ZZo8= | |
| AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://dev_airflow_user:airflow_password@postgres:25432/dev_airflow | |
| AIRFLOW__WEBSERVER__SECRET_KEY: 79197e3bef40d1d2a6cf07cb1f4c9f54e82a3bcfc6b92ed06d081856eee2 | |
| AWS_ACCESS_KEY_ID: minio-user | |
| AWS_SECRET_ACCESS_KEY: minio-password | |
| AWS_DEFAULT_REGION: us-east-1 | |
| ENV: dev | |
| FLOWER_PORT: 28888 | |
| FLOWER_PASSWORD: flowerpass | |
| MINIO_ROOT_USER: minio-user | |
| MINIO_ROOT_PASSWORD: minio-password | |
| MINIO_PORT_1: 9000 | |
| MINIO_PORT_2: 9090 | |
| POSTGRES_HOST_UID: 1001 | |
| POSTGRES_HOST_GID: 121 | |
| POSTGRES_HOST: postgres | |
| POSTGRES_PORT: 25432 | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_EPIGRAPH_HOST: postgres | |
| POSTGRES_EPIGRAPH_PORT: 25432 | |
| POSTGRES_EPIGRAPH_USER: dev_epigraph | |
| POSTGRES_EPIGRAPH_PASSWORD: dev_epigraph | |
| POSTGRES_EPIGRAPH_DB: dev_epigraphhub | |
| POSTGRES_EPIGRAPH_DB_PRIVATE: dev_privatehub | |
| POSTGRES_EPIGRAPH_DB_SANDBOX: dev_sandbox | |
| REDIS_HOST: redis | |
| REDIS_PORT: 6379 | |
| REDIS_PASSWORD: redispass | |
| SUPERSET_PORT: 8088 | |
| SUPERSET_DB_PATH_DIR_HOST: /tmp/superset | |
| SUPERSET_DB_PATH_HOST: /tmp/superset/superset.db | |
| SUPERSET_HOME: /opt/data/superset | |
| SUPERSET_MAIL_DEFAULT_SENDER: | |
| SUPERSET_MAIL_PASSWORD: | |
| SUPERSET_MAIL_PORT: 587 | |
| SUPERSET_MAIL_SERVER: | |
| SUPERSET_MAIL_USERNAME: | |
| SUPERSET_MAIL_USE_TLS: true | |
| SUPERSET_SECRET_KEY: superset-on-ci | |
| POSTGRES_DATA_DIR_HOST: /home/runner/work/psql_data | |
| POSTGRES_CONFIG_FILE_HOST: /home/runner/work/EpiGraphHub/EpiGraphHub/containers/postgresql/sql/dev/postgresql.conf | |
| SUPERSET_DB_USER: postgres | |
| SUPERSET_DB_PASSWORD: postgres | |
| SUPERSET_DB_DATABASE: postgres | |
| jobs: | |
| main: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 55 | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create environment variables file | |
| run: | | |
| export HOST_UID=$(id -u) | |
| export HOST_GID=$(id -g) | |
| export POSTGRES_HOST_UID=$(id -u) | |
| export POSTGRES_HOST_GID=$(id -g) | |
| envsubst < .env.tpl > .env | |
| mkdir /home/runner/work/psql_data | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniconda-version: "latest" | |
| environment-file: conda/base.yaml | |
| channels: conda-forge,r,nodefaults | |
| activate-environment: epigraphhub | |
| auto-update-conda: true | |
| conda-solver: libmamba | |
| - name: setup | |
| run: makim host.prepare | |
| - name: configure epigraphhub credentials | |
| run: | | |
| epigraphhub-config --name ci-epigraphhub \ | |
| --db-uri "${POSTGRES_EPIGRAPH_USER}:${POSTGRES_EPIGRAPH_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_EPIGRAPH_DB}" | |
| - name: configure aws s3 credentials to minio | |
| run: makim aws.create-s3-credential | |
| - name: build docker containers | |
| run: sugar build | |
| - name: start services | |
| run: | | |
| sugar ext restart --services superset,airflow,postgres --options -d | |
| - name: wait for the services are properly working | |
| run: | | |
| sugar ps | |
| makim containers.wait-all | |
| - name: run epigraphhub tests | |
| run: | | |
| wget https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/116.0.5845.96/linux64/chromedriver-linux64.zip -O /tmp/chromedriver-linux64.zip | |
| unzip /tmp/chromedriver-linux64.zip -d /tmp | |
| sudo rm /usr/bin/chromedriver | |
| sudo mv /tmp/chromedriver-linux64/chromedriver /usr/bin | |
| python -m unittest /home/runner/work/EpiGraphHub/EpiGraphHub/tests/* | |
| # TODO: Fix containers/postgres/sql/dev/02-epigraphhub.sql script | |
| # - name: Trigger all DAGs | |
| # run: | | |
| # make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test owid 2022-10-25' | |
| # make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test foph 2022-10-25' | |
| # make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test colombia 2022-10-25' | |
| # make containers-exec ARGS="-T" SERVICE=airflow CMD='airflow dags test web_status_test 2022-10-25' | |
| - name: Test MinIO access | |
| run: aws --endpoint-url http://localhost:9000/ s3 ls | |
| - name: Generate logs | |
| if: ${{ failure() }} | |
| run: | | |
| sugar logs --options --tail 1000 > /tmp/containers-services.log | |
| - name: Archive log artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: containers-services | |
| path: /tmp/containers-services.log |