add scrapy and beautifulsoup as dependencies #88
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a single version of Python | |
| # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python | |
| name: Unit Test Prefect Flows | |
| on: | |
| workflow_dispatch: # Allows for manual runs | |
| push: | |
| branches: | |
| - develop | |
| - test | |
| pull_request: | |
| branches: | |
| - develop | |
| - test | |
| permissions: | |
| contents: read | |
| jobs: | |
| install-python: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10"] # Can check other versions as well | |
| poetry-version: ["1.5.1"] # Can check other versions as well | |
| os: [ubuntu-latest] # Add [window-latest, macos-latest] later | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install poetry | |
| uses: abatilo/actions-poetry@v2 | |
| with: | |
| poetry-version: ${{ matrix.poetry-version }} | |
| - name: Setup a local virtual environment (if no poetry.toml file) | |
| run: | | |
| poetry config virtualenvs.create true --local | |
| poetry config virtualenvs.in-project true --local | |
| - uses: actions/cache@v3 | |
| name: Define a cache for the virtual environment based on the dependencies lock file | |
| with: | |
| path: ./.venv | |
| key: venv-${{ hashFiles('poetry.lock') }} | |
| - name: Install the project dependencies | |
| run: poetry install | |
| - name: Login into Prefect Cloud Server | |
| run: | | |
| poetry run prefect config set PREFECT_API_KEY=${{ secrets.PREFECT_API_KEY }} | |
| poetry run prefect config set PREFECT_API_URL=${{ secrets.PREFECT_API_URL }} | |
| - name: Run the unit tests | |
| run: poetry run pytest -v |