tests.yml

# This workflow will install Python dependencies and run tests
# For more information see:
# https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: tests

on:  # yamllint disable-line rule:truthy
  push:
    branches: ["main"]
  pull_request:
    branches: ["main"]
  workflow_dispatch:
    inputs:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        poetry-version: ["1.8.2"]
        node_version: ["lts/*"]

    steps:
      # Setup
      - name: Check out source repository
        uses: actions/checkout@v4

      - name: Set up poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: ${{ matrix.poetry-version }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'poetry'

      - name: Install Python dependencies
        run: poetry install --with daq,web,ana,dev

      # File filters
      - name: Paths changes filter
        uses: dorny/paths-filter@v3
        id: filter
        with:
          list-files: shell
          filters: |
            poetry:
            # Do not run poetry check and lock if just the pyproject.toml file changes
            # - added|modified: '**/pyproject.toml'
            - added|modified: '**/poetry.lock'
            python:
            - added|modified: '**.py'
            - added|modified: '**.ipynb'
            python_alone:
            - added|modified: '**.py'
            javascript:
            - added|modified: '**.js'
            html:
            - added|modified: '**.html'
            prettier:
            - added|modified: '**.html'
            - added|modified: '**.css'
            - added|modified: '**.scss'
            - added|modified: '**.json'
            # prettier-plugin-toml is not being properly installed on the actions runner,
            # lint toml files locally
            # - added|modified: '**.toml'
            yaml:
            - added|modified: '**.yaml'
            - added|modified: '**.yml'
            makefile:
            - added|modified: '**/Makefile'
            - added|modified: '**/makefile'
            markdown:
            - added|modified: '**.md'
            tex:
            - added|modified: '**.tex'
            rst:
            - added|modified: '**.rst'
            any:
            - added|modified: '**'

      - name: Free disk space
        uses: jlumbroso/free-disk-space@main
        with:
          # This might remove tools that are actually needed, if set to "true" but frees about 6 GB
          tool-cache: false

          # Does not break anything, and saves a lot of space, but runs slowly,
          # do not use unless necessary, i.e. when running poetry checks.
          large-packages: ${{steps.filter.outputs.poetry}}

          # Needed by any dockerized GitHub action
          docker-images: false

          # Gets the poetry cache back
          swap-storage: false

          # These will be cleaned
          android: true
          dotnet: true
          haskell: true

      # Filtered setup
      - name: Install Node
        if: steps.filter.outputs.javascript == 'true' || steps.filter.outputs.prettier == 'true'
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node_version }}
          check-latest: true

      - name: Install Node dependencies
        if: steps.filter.outputs.javascript == 'true' || steps.filter.outputs.prettier == 'true'
        run: sudo npm install --global standard@latest prettier@latest

      # Use "success() || failure()" to run a step even if a previous step fails
      # https://stackoverflow.com/a/58859404

      # Python

      - name: poetry
        if: (success() || failure()) && steps.filter.outputs.poetry == 'true'
        run: poetry check && poetry lock --no-update

      - name: isort
        if: (success() || failure()) && steps.filter.outputs.python == 'true'
        run: poetry run isort ${{ steps.filter.outputs.python_files }}

      - name: pyupgrade
        if: (success() || failure()) && steps.filter.outputs.python == 'true'
        run: poetry run pyupgrade ${{ steps.filter.outputs.python_files }}

      - name: black
        if: (success() || failure()) && steps.filter.outputs.python == 'true'
        run: poetry run black --check --diff ${{ steps.filter.outputs.python_files }}

      - name: blacken-docs
        if: (success() || failure()) && (steps.filter.outputs.python == 'true' || steps.filter.outputs.markdown == 'true' || steps.filter.outputs.rst == 'true' || steps.filter.outputs.tex == 'true')  # yamllint disable-line rule:line-length
        run: poetry run blacken-docs --line-length=100 --target-version=py311 ${{ steps.filter.outputs.python_files }} ${{ steps.filter.outputs.markdown_files }} ${{ steps.filter.outputs.rst_files }} ${{ steps.filter.outputs.tex_files }}  # yamllint disable-line rule:line-length

      - name: flake8
        if: (success() || failure()) && steps.filter.outputs.python == 'true'
        run: |
            echo "::group::version and plugins"
            poetry run flake8 --version
            echo "::endgroup::"
            poetry run flake8 --format github ${{ steps.filter.outputs.python_files }}

      # Disable: flake8-markdown does not use the pyproject.toml config correctly
      # - name: flake8-markdown
      #   if: (success() || failure()) && steps.filter.outputs.markdown == 'true'
      #   run: poetry run flake8-markdown ${{ steps.filter.outputs.markdown_files }}

      # https://pylint.pycqa.org/en/latest/user_guide/usage/output.html
      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
      # yamllint disable rule:line-length
      - name: pylint
        if: (success() || failure()) && steps.filter.outputs.python_alone == 'true'
        run: |
          poetry run pylint --msg-template='::error file={path},line={line},endLine={end_line},col={column},endColumn={end_column},title={symbol} ({msg_id})::{obj}: {msg}' ${{ steps.filter.outputs.python_alone_files }}
      # yamllint enable rule:line-length

      - name: mypy
        if: (success() || failure()) && steps.filter.outputs.python == 'true'
        run: poetry run mypy ${{ steps.filter.outputs.python_files }}

      - name: bandit
        if: (success() || failure()) && steps.filter.outputs.python == 'true'
        run: poetry run bandit -r ${{ steps.filter.outputs.python_files }}

      - name: vulture
        if: (success() || failure()) && steps.filter.outputs.python == 'true'
        run: poetry run vulture .  # Needs to run on whole package

      - name: deptry
        if: (success() || failure()) && steps.filter.outputs.python == 'true'
        run: poetry run deptry .  # Runs on whole package

      # Javascript

      - name: standard
        if: (success() || failure()) && steps.filter.outputs.javascript == 'true'
        run: standard ${{ steps.filter.outputs.javascript_files }}

      # HTML

      - name: html5validator
        if: (success() || failure()) && steps.filter.outputs.html == 'true'
        uses: Cyb3r-Jak3/html5validator-action@master  # blocklint: pragma
        with:
          config: .dev_config/.html5validator.yaml
          # only configurable to run on paths, so just run on whole package

      # Prettier: HTML CSS SCSS JSON (TOML plugin broken in GitHub actions)

      - name: prettier
        if: (success() || failure()) && steps.filter.outputs.prettier == 'true'
        id: prettier
        run: prettier --ignore-path .dev_config/.prettierignore --ignore-path .gitignore --config .dev_config/.prettierrc.yaml --check ${{ steps.filter.outputs.prettier_files }}  # yamllint disable-line rule:line-length

      # YAML

      - name: yamllint
        if: (success() || failure()) && steps.filter.outputs.yaml == 'true'
        run: poetry run yamllint --format github -c .dev_config/.yamllint.yaml --strict ${{ steps.filter.outputs.yaml_files }}  # yamllint disable-line rule:line-length

      # Makefile

      - name: checkmake
        if: (success() || failure()) && steps.filter.outputs.makefile == 'true'
        uses: Uno-Takashi/checkmake-action@main  # Can not pass multiple files

      # Shell scripts - no extension to filter on, always run

      - name: shellcheck shfmt checkbashisms
        if: success() || failure()
        uses: luizm/action-sh-checker@master  # blocklint: pragma
        env:
          SHFMT_OPTS: -bn -ci -sr -s -d
        with:
          sh_checker_checkbashisms_enable: true

      # Markdown

      - name: markdownlint
        if: (success() || failure()) && steps.filter.outputs.markdown == 'true'
        uses: nosborn/github-action-markdown-cli@v3.3.0
        with:
          files: ${{ steps.filter.outputs.markdown_files }}
          dot: true
          config_file: .dev_config/.markdownlint.yaml
          ignore_files: LICENSE.md

      # All files

      - name: detect-secrets
        if: success() || failure()
        run: poetry run detect-secrets-hook --exclude-lines 'integrity='

      - name: blocklint
        if: success() || failure()
        run: poetry run blocklint --skip-files=poetry.lock --max-issue-threshold=1

      - name: typos
        if: success() || failure()
        uses: crate-ci/typos@v1.21.0
        with:
          config: pyproject.toml

      - name: Trailing spaces
        if: (success() || failure()) && steps.filter.outputs.any == 'true'
        run: |
          TRAILING_SPACES=$(grep -rInH ' $' ${{ steps.filter.outputs.any_files }} || true)
          if [[ $TRAILING_SPACES ]]; then
            echo "::error Trailing spaces found, please remove them::"
            echo "$TRAILING_SPACES"
            false
          else
            echo "No trailing spaces found."
            true
          fi

      - name: DEV noqa via tmp file
        if: (success() || failure()) && steps.filter.outputs.any == 'true'
        run: |
          TMP_FIND_NOQA_COMMENTS=$(shell mktemp /tmp/find_noqa_comments.XXXXXX); \
          echo TMP_FIND_NOQA_COMMENTS || true; \
          echo $TMP_FIND_NOQA_COMMENTS || true; \
          echo $$TMP_FIND_NOQA_COMMENTS || true; \