testing.yml

# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Testing

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # Use the same Python version used the Dockerfile
        python-version: [3.9]

    steps:
      # Checkout and env setup
      - uses: actions/checkout@v2
        with:
          submodules: true

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-lint
          restore-keys: |
            ${{ runner.os }}-pip-lint

      # Lint things before going any further
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flake8 pytest pytest-cov

      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --show-source --statistics
          # exit-zero treats all errors as warnings.
          flake8 . --count --exit-zero --max-complexity=10 --statistics

  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # Use the same Python version used the Dockerfile
        python-version: [3.9]
        # app-context: ["codex", "mws"]
        app-context: ["codex"]
    env:
      OS: ubuntu-latest
      PYTHON: ${{ matrix.python-version }}
      TEST_DATABASE_URI: "postgresql://houston:development@db/houston_test"
      DOCKER_BUILDKIT: 1
      HOUSTON_APP_CONTEXT: ${{ matrix.app-context }}
    steps:
      # Checkout and env setup
      - uses: actions/checkout@v2

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}

      - uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install libmagic on Ubuntu
        run: |
          sudo apt install libmagic1

      - name: Setup App Context
        run: |
          set -ex
          ./scripts/${HOUSTON_APP_CONTEXT}/activate.sh

      - name: Run docker-compose (MWS)
        if: matrix.app-context == 'mws'
        run: |
          set -ex
          docker-compose up -d db redis elasticsearch elasticsearch2 elasticsearch3 sage sage-sync houston celery-beat celery-worker

      - name: Run docker-compose (Codex)
        if: matrix.app-context == 'codex'
        run: |
          set -ex
          docker-compose up -d db redis elasticsearch elasticsearch2 elasticsearch3 sage sage-sync houston celery-beat celery-worker
        env:
          ES_THRESHOLD: false

      - name: Check docker-compose status
        run: |
          set -ex
          # Check the state of the containers
          sleep 1m
          # Wait until houston is up
          while sleep 15
          do
            docker-compose logs houston | tail
            docker-compose ps
            if [ -n "$(docker-compose ps | grep Exit)" ]
            then
              exit 1
            fi
            wget --tries=1 -O - --header="Host: localhost:84" http://localhost:83/houston/ && break
          done

      - name: Run tests
        run: |
          set -ex
          docker-compose exec -T -e TEST_DATABASE_URI=$TEST_DATABASE_URI houston pytest --cov=./ --cov-append --random-order-seed=1

      - name: Check DB migrations (postgresql)
        if: matrix.app-context == 'codex'
        run: |
          set -ex
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston coverage run --append /usr/local/bin/invoke app.db.upgrade
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston coverage run --append /usr/local/bin/invoke app.db.downgrade
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston coverage run --append /usr/local/bin/invoke app.db.upgrade
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston coverage run --append /usr/local/bin/invoke app.db.downgrade --revision base
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston coverage run --append /usr/local/bin/invoke app.db.upgrade
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston coverage run --append /usr/local/bin/invoke app.db.migrate
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston /bin/bash -c 'if [ -n "$(git ls-files --others --exclude-standard migrations/versions/)" ]; then echo Missing database migration; exit 1; fi'
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston coverage run --append /usr/local/bin/invoke app.db._reset
        env:
          LOG_WIDTH: 120

      - name: Run other invoke tasks for coverage and errors
        if: matrix.app-context == 'codex'
        run: |
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston ./scripts/tests/run_tasks_for_coverage.sh
          docker-compose exec -T -e LOG_WIDTH=$LOG_WIDTH -e SQLALCHEMY_DATABASE_URI=$TEST_DATABASE_URI houston coverage xml
        env:
          LOG_WIDTH: 120

      - name: Upload coverage to Codecov
        if: matrix.app-context == 'codex'
        uses: codecov/codecov-action@v1.2.1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./_coverage/coverage.xml
          env_vars: OS,PYTHON,TEST_DATABASE_URI
          fail_ci_if_error: true

      - name: Show docker-compose logs if failed
        if: failure()
        run: |
          docker-compose logs --tail=250
          docker-compose ps

  build_frontend:
    name: Ensure frontend code builds
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
        with:
          submodules: true

      - name: Build the frontend
        run: |
          ./scripts/codex/build.frontend.sh

  build_swagger:
    name: Ensure swagger UI code builds
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2

      - name: Build the swagger UI
        run: |
          ./scripts/swagger/build.sh