Linting Updates

.dev_config/.vulture_ignore.py

@@ -1,26 +1,20 @@
 # flake8: noqa
 # pylint: skip-file
 # mypy: disable-error-code="name-defined"
-dfp_val  # unused variable (ana/exploratory_ana.py:341)
-_.converter  # unused attribute (daq/daq.py:145)
-dummy_signal  # unused variable (daq/daq.py:247)
-dummy_frame  # unused variable (daq/daq.py:248)
-_.when_held  # unused attribute (daq/daq.py:309)
-_.when_released  # unused attribute (daq/daq.py:310)
-disconnect  # unused function (daq/daq.py:490)
-_.show_server_banner  # unused attribute (daq/daq.py:511)
-args  # unused variable (daq/daq.py:511)
-pin  # unused variable (daq/pulse_test.py:19)
+dfp_val  # unused variable (ana/exploratory_ana.py:339)
+_.converter  # unused attribute (daq/daq.py:150)
+_.when_held  # unused attribute (daq/daq.py:310)
+_.when_released  # unused attribute (daq/daq.py:311)
+disconnect  # unused function (daq/daq.py:488)
+_.show_server_banner  # unused attribute (daq/daq.py:508)
+args  # unused variable (daq/daq.py:508)
 _.when_held  # unused attribute (daq/switch_test.py:33)
 _.when_released  # unused attribute (daq/switch_test.py:34)
-pin  # unused variable (fan_control/fan_control.py:76)
 pytorch_lightning  # unused import (TSModelWrappers/TSModelWrapper.py:22)
-_.on_exception  # unused method (TSModelWrappers/TSModelWrapper.py:170)
-trainer  # unused variable (TSModelWrappers/TSModelWrapper.py:172)
-pl_module  # unused variable (TSModelWrappers/TSModelWrapper.py:173)
-_.formatwarning  # unused attribute (utils/bayesian_opt.py:82)
-args  # unused variable (utils/bayesian_opt.py:82)
-dummy_signal  # unused variable (utils/bayesian_opt.py:763)
-dummy_frame  # unused variable (utils/bayesian_opt.py:764)
-_._iterations  # unused attribute (utils/bayesian_opt.py:931)
-_._previous_max  # unused attribute (utils/bayesian_opt.py:932)
+_.on_exception  # unused method (TSModelWrappers/TSModelWrapper.py:165)
+trainer  # unused variable (TSModelWrappers/TSModelWrapper.py:168)
+pl_module  # unused variable (TSModelWrappers/TSModelWrapper.py:169)
+_.formatwarning  # unused attribute (utils/bayesian_opt.py:87)
+args  # unused variable (utils/bayesian_opt.py:87)
+_._iterations  # unused attribute (utils/bayesian_opt.py:935)
+_._previous_max  # unused attribute (utils/bayesian_opt.py:936)

.gitattributes

@@ -1,5 +1,5 @@
-# ignore all ana ipynb and html files in diffs and github languages stats
+# ignore all ana/ ipynb and html files in diffs and GitHub languages stats
 ana/*.ipynb linguist-generated=true
 ana/*.html linguist-generated=true
-# ignore html media files in github languages stats
+# ignore media/ html files in GitHub languages stats
 media/**/*.html linguist-documentation=true

.github/pull_request_template.md

@@ -14,7 +14,7 @@
 # Checklist
 * [ ] I have performed a self-review of my own code
 * [ ] I have run the `pre-commit` checks
-* [ ] The [github `tests` action](https://github.com/mepland/chance_of_showers/actions/workflows/tests.yml) is passing
+* [ ] The [GitHub `tests` action](https://github.com/mepland/chance_of_showers/actions/workflows/tests.yml) is passing
 * [ ] I have commented my code, particularly in hard-to-understand areas
 * [ ] I have updated the documentation as required
 

.github/workflows/tests.yml

@@ -23,7 +23,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.11"]
-        poetry-version: ["1.8.2"]
+        poetry-version: ["1.8.3"]
         node_version: ["lts/*"]
 
     steps:
@@ -42,11 +42,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: 'poetry'
 
-      - name: Install Python Dependencies
+      - name: Install Python dependencies
         run: poetry install --with daq,web,ana,dev
 
       # File filters
-      - name: Paths Changes Filter
+      - name: Paths changes filter
         uses: dorny/paths-filter@v3
         id: filter
         with:
@@ -88,7 +88,7 @@ jobs:
             any:
             - added|modified: '**'
 
-      - name: Free Disk Space
+      - name: Free disk space
         uses: jlumbroso/free-disk-space@main
         with:
           # This might remove tools that are actually needed, if set to "true" but frees about 6 GB
@@ -98,7 +98,7 @@ jobs:
           # do not use unless necessary, i.e. when running poetry checks.
           large-packages: ${{steps.filter.outputs.poetry}}
 
-          # Needed by any dockerized github action
+          # Needed by any dockerized GitHub action
           docker-images: false
 
           # Gets the poetry cache back
@@ -117,28 +117,22 @@ jobs:
           node-version: ${{ matrix.node_version }}
           check-latest: true
 
-      - name: Install Node Dependencies
+      - name: Install Node dependencies
         if: steps.filter.outputs.javascript == 'true' || steps.filter.outputs.prettier == 'true'
-        run: sudo npm install --global standard prettier
+        run: sudo npm install --global standard@latest prettier@latest
 
-      # All files
-      - name: detect-secrets
-        if: success() || failure()
-        run: poetry run detect-secrets-hook --exclude-lines 'integrity='
+      # Use "success() || failure()" to run a step even if a previous step fails
+      # https://stackoverflow.com/a/58859404
 
-      - name: blocklint
-        if: success() || failure()
-        run: poetry run blocklint --skip-files=poetry.lock --max-issue-threshold=1
+      # Python
 
-      # poetry
       - name: poetry
         if: (success() || failure()) && steps.filter.outputs.poetry == 'true'
         run: poetry check && poetry lock --no-update
 
-      # python
       - name: isort
         if: (success() || failure()) && steps.filter.outputs.python == 'true'
-        run: poetry run isort ${{ steps.filter.outputs.python_files }}
+        run: poetry run isort --check-only --diff ${{ steps.filter.outputs.python_files }}
 
       - name: pyupgrade
         if: (success() || failure()) && steps.filter.outputs.python == 'true'
@@ -148,21 +142,42 @@ jobs:
         if: (success() || failure()) && steps.filter.outputs.python == 'true'
         run: poetry run black --check --diff ${{ steps.filter.outputs.python_files }}
 
+      # yamllint disable rule:line-length
+      - name: blacken-docs
+        if: (success() || failure()) && (steps.filter.outputs.python == 'true' || steps.filter.outputs.markdown == 'true' || steps.filter.outputs.rst == 'true' || steps.filter.outputs.tex == 'true')
+        run: poetry run blacken-docs --line-length=100 --target-version=py311 ${{ steps.filter.outputs.python_files }} ${{ steps.filter.outputs.markdown_files }} ${{ steps.filter.outputs.rst_files }} ${{ steps.filter.outputs.tex_files }}
+      # yamllint enable rule:line-length
+
       - name: flake8
         if: (success() || failure()) && steps.filter.outputs.python == 'true'
         run: |
             echo "::group::version and plugins"
             poetry run flake8 --version
             echo "::endgroup::"
-            poetry run flake8 ${{ steps.filter.outputs.python_files }}
+            poetry run flake8 --format github ${{ steps.filter.outputs.python_files }}
 
-      - name: mypy
+      # Disable: flake8-markdown does not use the pyproject.toml config correctly
+      # - name: flake8-markdown
+      #   if: (success() || failure()) && steps.filter.outputs.markdown == 'true'
+      #   run: poetry run flake8-markdown ${{ steps.filter.outputs.markdown_files }}
+
+      - name: ruff - not required
         if: (success() || failure()) && steps.filter.outputs.python == 'true'
-        run: poetry run mypy ${{ steps.filter.outputs.python_files }}
+        run: |
+            poetry run ruff check --exit-zero ${{ steps.filter.outputs.python_files }}
 
+      # https://pylint.pycqa.org/en/latest/user_guide/usage/output.html
+      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
+      # yamllint disable rule:line-length
       - name: pylint
         if: (success() || failure()) && steps.filter.outputs.python_alone == 'true'
-        run: poetry run pylint ${{ steps.filter.outputs.python_alone_files }}
+        run: |
+          poetry run pylint --msg-template='::error file={path},line={line},endLine={end_line},col={column},endColumn={end_column},title={symbol} ({msg_id})::{obj}: {msg}' ${{ steps.filter.outputs.python_alone_files }}
+      # yamllint enable rule:line-length
+
+      - name: mypy
+        if: (success() || failure()) && steps.filter.outputs.python == 'true'
+        run: poetry run mypy ${{ steps.filter.outputs.python_files }}
 
       - name: bandit
         if: (success() || failure()) && steps.filter.outputs.python == 'true'
@@ -176,36 +191,42 @@ jobs:
         if: (success() || failure()) && steps.filter.outputs.python == 'true'
         run: poetry run deptry .  # Runs on whole package
 
-      # javascript
+      # Javascript
+
       - name: standard
         if: (success() || failure()) && steps.filter.outputs.javascript == 'true'
         run: standard ${{ steps.filter.outputs.javascript_files }}
 
-      # html
+      # HTML
+
       - name: html5validator
         if: (success() || failure()) && steps.filter.outputs.html == 'true'
         uses: Cyb3r-Jak3/html5validator-action@master  # blocklint: pragma
         with:
           config: .dev_config/.html5validator.yaml
           # only configurable to run on paths, so just run on whole package
 
-      # prettier: html css scss json
+      # Prettier: HTML CSS SCSS JSON (TOML plugin broken in GitHub actions)
+
       - name: prettier
         if: (success() || failure()) && steps.filter.outputs.prettier == 'true'
         id: prettier
         run: prettier --ignore-path .dev_config/.prettierignore --ignore-path .gitignore --config .dev_config/.prettierrc.yaml --check ${{ steps.filter.outputs.prettier_files }}  # yamllint disable-line rule:line-length
 
-      # yaml
+      # YAML
+
       - name: yamllint
         if: (success() || failure()) && steps.filter.outputs.yaml == 'true'
-        run: poetry run yamllint -c .dev_config/.yamllint.yaml --strict ${{ steps.filter.outputs.yaml_files }}  # yamllint disable-line rule:line-length
+        run: poetry run yamllint --format github -c .dev_config/.yamllint.yaml --strict ${{ steps.filter.outputs.yaml_files }}  # yamllint disable-line rule:line-length
+
+      # Makefile
 
-      # makefile
       - name: checkmake
         if: (success() || failure()) && steps.filter.outputs.makefile == 'true'
         uses: Uno-Takashi/checkmake-action@main  # Can not pass multiple files
 
-      # shell files - no extension to filter on, always run
+      # Shell scripts - no extension to filter on, always run
+
       - name: shellcheck shfmt checkbashisms
         if: success() || failure()
         uses: luizm/action-sh-checker@master  # blocklint: pragma
@@ -214,14 +235,8 @@ jobs:
         with:
           sh_checker_checkbashisms_enable: true
 
-      # typos - always run
-      - name: typos
-        if: success() || failure()
-        uses: crate-ci/[email protected]
-        with:
-          config: pyproject.toml
+      # Markdown
 
-      # docs
       - name: markdownlint
         if: (success() || failure()) && steps.filter.outputs.markdown == 'true'
         uses: nosborn/[email protected]
@@ -231,19 +246,78 @@ jobs:
           config_file: .dev_config/.markdownlint.yaml
           ignore_files: LICENSE.md
 
-      - name: blacken-docs
-        if: (success() || failure()) && (steps.filter.outputs.python == 'true' || steps.filter.outputs.markdown == 'true' || steps.filter.outputs.rst == 'true' || steps.filter.outputs.tex == 'true')  # yamllint disable-line rule:line-length
-        run: poetry run blacken-docs --line-length=100 ${{ steps.filter.outputs.python_files }} ${{ steps.filter.outputs.markdown_files }} ${{ steps.filter.outputs.rst_files }} ${{ steps.filter.outputs.tex_files }}  # yamllint disable-line rule:line-length
+      # All files
+
+      - name: detect-secrets
+        if: (success() || failure()) && steps.filter.outputs.any == 'true'
+        run: poetry run detect-secrets-hook --exclude-lines 'integrity=' ${{ steps.filter.outputs.any_files }}  # yamllint disable-line rule:line-length
+
+      - name: blocklint
+        if: (success() || failure()) && steps.filter.outputs.any == 'true'
+        run: poetry run blocklint --skip-files=poetry.lock --max-issue-threshold=1 ${{ steps.filter.outputs.any_files }}  # yamllint disable-line rule:line-length
+
+      - name: typos
+        if: (success() || failure()) && steps.filter.outputs.any == 'true'
+        uses: crate-ci/[email protected]
+        with:
+          config: pyproject.toml
+          files: ${{ steps.filter.outputs.any_files }}
 
-      - name: trailing-spaces
+      - name: Trailing spaces
         if: (success() || failure()) && steps.filter.outputs.any == 'true'
         run: |
           TRAILING_SPACES=$(grep -rInH ' $' ${{ steps.filter.outputs.any_files }} || true)
           if [[ $TRAILING_SPACES ]]; then
-            echo "Trailing spaces found, please remove them:"
+            echo "::error::Trailing spaces found, please remove them"
             echo "$TRAILING_SPACES"
             false
           else
             echo "No trailing spaces found."
             true
           fi
+
+      # yamllint disable rule:line-length
+      - name: noqa flags
+        if: (success() || failure()) && steps.filter.outputs.any == 'true'
+        run: |
+          TMP_FIND_NOQA_COMMENTS=$(mktemp /tmp/find_noqa_comments.XXXXXX)
+          # isort, flake8, ruff, pylint, mypy, bandit
+          if [[ ${{ steps.filter.outputs.python }} == 'true' ]]; then
+            grep -rInH 'isort:\|noqa\|pylint\| type:\|nosec' ${{ steps.filter.outputs.python_files }} >> $TMP_FIND_NOQA_COMMENTS || true
+          fi
+          # standard
+          if [[ ${{ steps.filter.outputs.javascript }} == 'true' ]]; then
+            grep -rInH 'eslint' ${{ steps.filter.outputs.javascript_files }} >> $TMP_FIND_NOQA_COMMENTS || true
+          fi
+          # prettier
+          if [[ ${{ steps.filter.outputs.prettier }} == 'true' ]]; then
+            grep -rInH 'prettier-ignore' ${{ steps.filter.outputs.prettier_files }} >> $TMP_FIND_NOQA_COMMENTS || true
+          fi
+          # yamllint
+          if [[ ${{ steps.filter.outputs.yaml }} == 'true' ]]; then
+            grep -rInH 'yamllint [de]' ${{ steps.filter.outputs.yaml_files }} >> $TMP_FIND_NOQA_COMMENTS || true
+          fi
+          # markdownlint
+          if [[ ${{ steps.filter.outputs.markdown }} == 'true' ]]; then
+            grep -rInH 'markdownlint-[de]' ${{ steps.filter.outputs.markdown_files }} >> $TMP_FIND_NOQA_COMMENTS || true
+          fi
+          # detect-secrets, blocklint
+          grep -rInH 'pragma: allowlist\|blocklint:.*pragma' ${{ steps.filter.outputs.any_files }} >> $TMP_FIND_NOQA_COMMENTS || true
+          # Remove false positive lines from Makefile and .pre-commit-config.yaml
+          sed -i -r '/^Makefile:[0-9]+:(# detect-secrets ~|# blocklint ~|.*grep)/d' $TMP_FIND_NOQA_COMMENTS
+          # Remove code, keep comments
+          perl -pi -E 's/(^.*?:[0-9]+:).*?(#|<\!-- )/\1\2/g' $TMP_FIND_NOQA_COMMENTS
+          # Sort and remove duplicates
+          sort -u -o $TMP_FIND_NOQA_COMMENTS $TMP_FIND_NOQA_COMMENTS
+          # Print results
+          if [[ -s $TMP_FIND_NOQA_COMMENTS ]]; then
+            echo "::warning::The following lines contain noqa comments"
+            cat $TMP_FIND_NOQA_COMMENTS
+          else
+            echo "No noqa flags found."
+          fi
+          # Remove temporary file
+          rm -f $TMP_FIND_NOQA_COMMENTS
+          # Always pass, just warn
+          true
+      # yamllint enable rule:line-length