Linting Updates #397
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies and run tests | |
| # For more information see: | |
| # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python | |
| name: tests | |
| on: # yamllint disable-line rule:truthy | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| branches: ["main"] | |
| workflow_dispatch: | |
| inputs: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11"] | |
| poetry-version: ["1.8.2"] | |
| node_version: ["lts/*"] | |
| steps: | |
| # Setup | |
| - name: Check out source repository | |
| uses: actions/checkout@v4 | |
| - name: Set up poetry | |
| uses: abatilo/actions-poetry@v2 | |
| with: | |
| poetry-version: ${{ matrix.poetry-version }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'poetry' | |
| - name: Install Python dependencies | |
| run: poetry install --with daq,web,ana,dev | |
| # File filters | |
| - name: Paths changes filter | |
| uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| list-files: shell | |
| filters: | | |
| poetry: | |
| # Do not run poetry check and lock if just the pyproject.toml file changes | |
| # - added|modified: '**/pyproject.toml' | |
| - added|modified: '**/poetry.lock' | |
| python: | |
| - added|modified: '**.py' | |
| - added|modified: '**.ipynb' | |
| python_alone: | |
| - added|modified: '**.py' | |
| javascript: | |
| - added|modified: '**.js' | |
| html: | |
| - added|modified: '**.html' | |
| prettier: | |
| - added|modified: '**.html' | |
| - added|modified: '**.css' | |
| - added|modified: '**.scss' | |
| - added|modified: '**.json' | |
| # prettier-plugin-toml is not being properly installed on the actions runner, | |
| # lint toml files locally | |
| # - added|modified: '**.toml' | |
| yaml: | |
| - added|modified: '**.yaml' | |
| - added|modified: '**.yml' | |
| makefile: | |
| - added|modified: '**/Makefile' | |
| - added|modified: '**/makefile' | |
| markdown: | |
| - added|modified: '**.md' | |
| tex: | |
| - added|modified: '**.tex' | |
| rst: | |
| - added|modified: '**.rst' | |
| any: | |
| - added|modified: '**' | |
| # - name: Free disk space | |
| # uses: jlumbroso/free-disk-space@main | |
| # with: | |
| # # This might remove tools that are actually needed, if set to "true" but frees about 6 GB | |
| # tool-cache: false | |
| # # Does not break anything, and saves a lot of space, but runs slowly, | |
| # # do not use unless necessary, i.e. when running poetry checks. | |
| # large-packages: ${{steps.filter.outputs.poetry}} | |
| # # Needed by any dockerized GitHub action | |
| # docker-images: false | |
| # # Gets the poetry cache back | |
| # swap-storage: false | |
| # # These will be cleaned | |
| # android: true | |
| # dotnet: true | |
| # haskell: true | |
| # # Filtered setup | |
| # - name: Install Node | |
| # if: steps.filter.outputs.javascript == 'true' || steps.filter.outputs.prettier == 'true' | |
| # uses: actions/setup-node@v4 | |
| # with: | |
| # node-version: ${{ matrix.node_version }} | |
| # check-latest: true | |
| # - name: Install Node dependencies | |
| # if: steps.filter.outputs.javascript == 'true' || steps.filter.outputs.prettier == 'true' | |
| # run: sudo npm install --global standard@latest prettier@latest | |
| # # Use "success() || failure()" to run a step even if a previous step fails | |
| # # https://stackoverflow.com/a/58859404 | |
| # # Python | |
| # - name: poetry | |
| # if: (success() || failure()) && steps.filter.outputs.poetry == 'true' | |
| # run: poetry check && poetry lock --no-update | |
| # - name: isort | |
| # if: (success() || failure()) && steps.filter.outputs.python == 'true' | |
| # run: poetry run isort ${{ steps.filter.outputs.python_files }} | |
| # - name: pyupgrade | |
| # if: (success() || failure()) && steps.filter.outputs.python == 'true' | |
| # run: poetry run pyupgrade ${{ steps.filter.outputs.python_files }} | |
| # - name: black | |
| # if: (success() || failure()) && steps.filter.outputs.python == 'true' | |
| # run: poetry run black --check --diff ${{ steps.filter.outputs.python_files }} | |
| # - name: blacken-docs | |
| # if: (success() || failure()) && (steps.filter.outputs.python == 'true' || steps.filter.outputs.markdown == 'true' || steps.filter.outputs.rst == 'true' || steps.filter.outputs.tex == 'true') # yamllint disable-line rule:line-length | |
| # run: poetry run blacken-docs --line-length=100 --target-version=py311 ${{ steps.filter.outputs.python_files }} ${{ steps.filter.outputs.markdown_files }} ${{ steps.filter.outputs.rst_files }} ${{ steps.filter.outputs.tex_files }} # yamllint disable-line rule:line-length | |
| # - name: flake8 | |
| # if: (success() || failure()) && steps.filter.outputs.python == 'true' | |
| # run: | | |
| # echo "::group::version and plugins" | |
| # poetry run flake8 --version | |
| # echo "::endgroup::" | |
| # poetry run flake8 --format github ${{ steps.filter.outputs.python_files }} | |
| # # Disable: flake8-markdown does not use the pyproject.toml config correctly | |
| # # - name: flake8-markdown | |
| # # if: (success() || failure()) && steps.filter.outputs.markdown == 'true' | |
| # # run: poetry run flake8-markdown ${{ steps.filter.outputs.markdown_files }} | |
| # # https://pylint.pycqa.org/en/latest/user_guide/usage/output.html | |
| # # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message | |
| # # yamllint disable rule:line-length | |
| # - name: pylint | |
| # if: (success() || failure()) && steps.filter.outputs.python_alone == 'true' | |
| # run: | | |
| # poetry run pylint --msg-template='::error file={path},line={line},endLine={end_line},col={column},endColumn={end_column},title={symbol} ({msg_id})::{obj}: {msg}' ${{ steps.filter.outputs.python_alone_files }} | |
| # # yamllint enable rule:line-length | |
| # - name: mypy | |
| # if: (success() || failure()) && steps.filter.outputs.python == 'true' | |
| # run: poetry run mypy ${{ steps.filter.outputs.python_files }} | |
| # - name: bandit | |
| # if: (success() || failure()) && steps.filter.outputs.python == 'true' | |
| # run: poetry run bandit -r ${{ steps.filter.outputs.python_files }} | |
| # - name: vulture | |
| # if: (success() || failure()) && steps.filter.outputs.python == 'true' | |
| # run: poetry run vulture . # Needs to run on whole package | |
| # - name: deptry | |
| # if: (success() || failure()) && steps.filter.outputs.python == 'true' | |
| # run: poetry run deptry . # Runs on whole package | |
| # # Javascript | |
| # - name: standard | |
| # if: (success() || failure()) && steps.filter.outputs.javascript == 'true' | |
| # run: standard ${{ steps.filter.outputs.javascript_files }} | |
| # # HTML | |
| # - name: html5validator | |
| # if: (success() || failure()) && steps.filter.outputs.html == 'true' | |
| # uses: Cyb3r-Jak3/html5validator-action@master # blocklint: pragma | |
| # with: | |
| # config: .dev_config/.html5validator.yaml | |
| # # only configurable to run on paths, so just run on whole package | |
| # # Prettier: HTML CSS SCSS JSON (TOML plugin broken in GitHub actions) | |
| # - name: prettier | |
| # if: (success() || failure()) && steps.filter.outputs.prettier == 'true' | |
| # id: prettier | |
| # run: prettier --ignore-path .dev_config/.prettierignore --ignore-path .gitignore --config .dev_config/.prettierrc.yaml --check ${{ steps.filter.outputs.prettier_files }} # yamllint disable-line rule:line-length | |
| # # YAML | |
| # - name: yamllint | |
| # if: (success() || failure()) && steps.filter.outputs.yaml == 'true' | |
| # run: poetry run yamllint --format github -c .dev_config/.yamllint.yaml --strict ${{ steps.filter.outputs.yaml_files }} # yamllint disable-line rule:line-length | |
| # # Makefile | |
| # - name: checkmake | |
| # if: (success() || failure()) && steps.filter.outputs.makefile == 'true' | |
| # uses: Uno-Takashi/checkmake-action@main # Can not pass multiple files | |
| # # Shell scripts - no extension to filter on, always run | |
| # - name: shellcheck shfmt checkbashisms | |
| # if: success() || failure() | |
| # uses: luizm/action-sh-checker@master # blocklint: pragma | |
| # env: | |
| # SHFMT_OPTS: -bn -ci -sr -s -d | |
| # with: | |
| # sh_checker_checkbashisms_enable: true | |
| # # Markdown | |
| # - name: markdownlint | |
| # if: (success() || failure()) && steps.filter.outputs.markdown == 'true' | |
| # uses: nosborn/[email protected] | |
| # with: | |
| # files: ${{ steps.filter.outputs.markdown_files }} | |
| # dot: true | |
| # config_file: .dev_config/.markdownlint.yaml | |
| # ignore_files: LICENSE.md | |
| # # All files | |
| # - name: detect-secrets | |
| # if: success() || failure() | |
| # run: poetry run detect-secrets-hook --exclude-lines 'integrity=' | |
| # - name: blocklint | |
| # if: success() || failure() | |
| # run: poetry run blocklint --skip-files=poetry.lock --max-issue-threshold=1 | |
| # - name: typos | |
| # if: success() || failure() | |
| # uses: crate-ci/[email protected] | |
| # with: | |
| # config: pyproject.toml | |
| # - name: Trailing spaces | |
| # if: (success() || failure()) && steps.filter.outputs.any == 'true' | |
| # run: | | |
| # TRAILING_SPACES=$(grep -rInH ' $' ${{ steps.filter.outputs.any_files }} || true) | |
| # if [[ $TRAILING_SPACES ]]; then | |
| # echo "::error Trailing spaces found, please remove them::" | |
| # echo "$TRAILING_SPACES" | |
| # false | |
| # else | |
| # echo "No trailing spaces found." | |
| # true | |
| # fi | |
| - name: DEV noqa via tmp file | |
| if: (success() || failure()) && steps.filter.outputs.any == 'true' | |
| run: | | |
| TMP_FIND_NOQA_COMMENTS=$(mktemp /tmp/find_noqa_comments.XXXXXX) | |
| # isort, flake8, ruff, pylint, mypy, bandit | |
| grep -rInH 'isort:\|noqa\|pylint\| type:\|nosec' ${{ steps.filter.outputs.python_files }} >> $TMP_FIND_NOQA_COMMENTS || true | |
| # standard | |
| grep -rInH 'eslint' ${{ steps.filter.outputs.javascript_files }} >> $TMP_FIND_NOQA_COMMENTS || true | |
| # prettier | |
| grep -rInH 'prettier-ignore' ${{ steps.filter.outputs.prettier_files }} >> $TMP_FIND_NOQA_COMMENTS || true | |
| # yamllint | |
| grep -rInH 'yamllint [de]' ${{ steps.filter.outputs.yaml_files }} >> $TMP_FIND_NOQA_COMMENTS || true | |
| # markdownlint | |
| grep -rInH 'markdownlint-[de]' ${{ steps.filter.outputs.markdown_files }} >> $TMP_FIND_NOQA_COMMENTS || true | |
| # detect-secrets, blocklint | |
| # TODO | |
| # grep -rInH 'pragma: allowlist\|blocklint:.*pragma' ${{ steps.filter.outputs.any_files }} >> $TMP_FIND_NOQA_COMMENTS || true | |
| # Remove false positive lines from Makefile and .pre-commit-config.yaml | |
| sed -i -r '/^Makefile:[0-9]+:(# detect-secrets ~|# blocklint ~|.*grep)/d' $TMP_FIND_NOQA_COMMENTS | |
| # Remove code, keep comments | |
| # perl -pi -E 's/(^.*?:[0-9]+:).*?(#|<\!-- )/\1\2/g' $TMP_FIND_NOQA_COMMENTS | |
| # Sort and remove duplicates | |
| # sort -u -o $TMP_FIND_NOQA_COMMENTS $TMP_FIND_NOQA_COMMENTS | |
| # print results | |
| echo "hard cat" | |
| cat $TMP_FIND_NOQA_COMMENTS | |
| echo "end hard cat" | |
| if [[ -s $TMP_FIND_NOQA_COMMENTS ]]; then | |
| echo "-s is true" | |
| echo "::warning The following lines contain noqa comments::" | |
| cat $TMP_FIND_NOQA_COMMENTS | |
| else | |
| echo "-s is false" | |
| echo "No noqa flags found." | |
| fi | |
| # remove temporary file | |
| rm -f $TMP_FIND_NOQA_COMMENTS | |
| # Always pass, just warn | |
| true |