From a28c32a21b303eea396490445a017c630b808ded Mon Sep 17 00:00:00 2001 From: Matt Epland Date: Mon, 20 May 2024 22:10:02 -0400 Subject: [PATCH] Infra improvements (#69) * Specify python3.11 in setup * Run all tests even if prior tests fail * Reorder pyupgrade after isort * Add log2ram to README * Add trailing-spaces check * find trailing spaces in Makefile * Always print filenames via H * Fix double space --- .github/workflows/tests.yml | 59 ++++++++++++++++++++++++------------- .pre-commit-config.yaml | 12 ++++---- Makefile | 29 +++++++++++------- README.md | 9 +++++- pyproject.toml | 2 +- 5 files changed, 72 insertions(+), 39 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d806830..856bda82 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -85,6 +85,8 @@ jobs: - added|modified: '**.tex' rst: - added|modified: '**.rst' + any: + - added|modified: '**' - name: Free Disk Space uses: jlumbroso/free-disk-space@main @@ -121,27 +123,33 @@ jobs: # All files - name: detect-secrets + if: success() || failure() run: poetry run detect-secrets-hook --exclude-lines 'integrity=' - name: blocklint + if: success() || failure() run: poetry run blocklint --skip-files=poetry.lock --max-issue-threshold=1 # poetry - name: poetry - if: steps.filter.outputs.poetry == 'true' + if: (success() || failure()) && steps.filter.outputs.poetry == 'true' run: poetry check && poetry lock --no-update # python - name: isort - if: steps.filter.outputs.python == 'true' + if: (success() || failure()) && steps.filter.outputs.python == 'true' run: poetry run isort ${{ steps.filter.outputs.python_files }} + - name: pyupgrade + if: (success() || failure()) && steps.filter.outputs.python == 'true' + run: poetry run pyupgrade ${{ steps.filter.outputs.python_files }} + - name: black - if: steps.filter.outputs.python == 'true' + if: (success() || failure()) && steps.filter.outputs.python == 'true' run: poetry run black --check --diff ${{ steps.filter.outputs.python_files }} - name: flake8 - if: steps.filter.outputs.python == 'true' + if: (success() || failure()) && steps.filter.outputs.python == 'true' run: | echo "::group::version and plugins" poetry run flake8 --version @@ -149,37 +157,33 @@ jobs: poetry run flake8 ${{ steps.filter.outputs.python_files }} - name: mypy - if: steps.filter.outputs.python == 'true' + if: (success() || failure()) && steps.filter.outputs.python == 'true' run: poetry run mypy ${{ steps.filter.outputs.python_files }} - name: pylint - if: steps.filter.outputs.python_alone == 'true' + if: (success() || failure()) && steps.filter.outputs.python_alone == 'true' run: poetry run pylint ${{ steps.filter.outputs.python_alone_files }} - name: bandit - if: steps.filter.outputs.python == 'true' + if: (success() || failure()) && steps.filter.outputs.python == 'true' run: poetry run bandit -r ${{ steps.filter.outputs.python_files }} - name: vulture - if: steps.filter.outputs.python == 'true' + if: (success() || failure()) && steps.filter.outputs.python == 'true' run: poetry run vulture . # Needs to run on whole package - - name: pyupgrade - if: steps.filter.outputs.python == 'true' - run: poetry run pyupgrade ${{ steps.filter.outputs.python_files }} - - name: deptry - if: steps.filter.outputs.python == 'true' + if: (success() || failure()) && steps.filter.outputs.python == 'true' run: poetry run deptry . # Runs on whole package # javascript - name: standard - if: steps.filter.outputs.javascript == 'true' + if: (success() || failure()) && steps.filter.outputs.javascript == 'true' run: standard ${{ steps.filter.outputs.javascript_files }} # html - name: html5validator - if: steps.filter.outputs.html == 'true' + if: (success() || failure()) && steps.filter.outputs.html == 'true' uses: Cyb3r-Jak3/html5validator-action@master # blocklint: pragma with: config: .dev_config/.html5validator.yaml @@ -187,22 +191,23 @@ jobs: # prettier: html css scss json - name: prettier - if: steps.filter.outputs.prettier == 'true' + if: (success() || failure()) && steps.filter.outputs.prettier == 'true' id: prettier run: prettier --ignore-path .dev_config/.prettierignore --ignore-path .gitignore --config .dev_config/.prettierrc.yaml --check ${{ steps.filter.outputs.prettier_files }} # yamllint disable-line rule:line-length # yaml - name: yamllint - if: steps.filter.outputs.yaml == 'true' + if: (success() || failure()) && steps.filter.outputs.yaml == 'true' run: poetry run yamllint -c .dev_config/.yamllint.yaml --strict ${{ steps.filter.outputs.yaml_files }} # yamllint disable-line rule:line-length # makefile - name: checkmake - if: steps.filter.outputs.makefile == 'true' + if: (success() || failure()) && steps.filter.outputs.makefile == 'true' uses: Uno-Takashi/checkmake-action@main # Can not pass multiple files # shell files - no extension to filter on, always run - name: shellcheck shfmt checkbashisms + if: success() || failure() uses: luizm/action-sh-checker@master # blocklint: pragma env: SHFMT_OPTS: -bn -ci -sr -s -d @@ -211,13 +216,14 @@ jobs: # typos - always run - name: typos + if: success() || failure() uses: crate-ci/typos@v1.19.0 with: config: pyproject.toml # docs - name: markdownlint - if: steps.filter.outputs.markdown == 'true' + if: (success() || failure()) && steps.filter.outputs.markdown == 'true' uses: nosborn/github-action-markdown-cli@v3.3.0 with: files: ${{ steps.filter.outputs.markdown_files }} @@ -226,5 +232,18 @@ jobs: ignore_files: LICENSE.md - name: blacken-docs - if: steps.filter.outputs.python == 'true' || steps.filter.outputs.markdown == 'true' || steps.filter.outputs.rst == 'true' || steps.filter.outputs.tex == 'true' # yamllint disable-line rule:line-length + if: (success() || failure()) && (steps.filter.outputs.python == 'true' || steps.filter.outputs.markdown == 'true' || steps.filter.outputs.rst == 'true' || steps.filter.outputs.tex == 'true') # yamllint disable-line rule:line-length run: poetry run blacken-docs --line-length=100 ${{ steps.filter.outputs.python_files }} ${{ steps.filter.outputs.markdown_files }} ${{ steps.filter.outputs.rst_files }} ${{ steps.filter.outputs.tex_files }} # yamllint disable-line rule:line-length + + - name: trailing-spaces + if: (success() || failure()) && steps.filter.outputs.any == 'true' + run: | + TRAILING_SPACES=$(grep -rInH ' $' ${{ steps.filter.outputs.any_files }} || true) + if [[ $TRAILING_SPACES ]]; then + echo "Trailing spaces found, please remove them:" + echo "$TRAILING_SPACES" + false + else + echo "No trailing spaces found." + true + fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6b35c6b..41d65195 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,12 @@ repos: hooks: - id: isort + # pyupgrade + - repo: https://github.com/asottile/pyupgrade + rev: 'v3.15.1' + hooks: + - id: pyupgrade + # black - repo: https://github.com/psf/black rev: '24.3.0' @@ -115,12 +121,6 @@ repos: hooks: - id: vulture - # pyupgrade - - repo: https://github.com/asottile/pyupgrade - rev: 'v3.15.1' - hooks: - - id: pyupgrade - # deptry - only works when run within poetry env, but can't ensure from within pre-commit # - repo: https://github.com/fpgmaas/deptry.git # rev: "0.12.0" diff --git a/Makefile b/Makefile index d1b13830..6047b1e2 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,13 @@ .PHONY: setupDAQ setupDAQ: + @poetry env use python3.11 @poetry install --with daq,web,dev --no-root @poetry run pre-commit install .PHONY: setupANA setupANA: + @poetry env use python3.11 @poetry install --with ana,dev --no-root @poetry run pre-commit install @@ -31,6 +33,10 @@ pre-commit-update: isort: @poetry run isort . +.PHONY: pyupgrade +pyupgrade: + @poetry run pyupgrade $(shell git ls-files '*.py' '*.ipynb') + .PHONY: black black: @poetry run black . @@ -80,10 +86,6 @@ vulture-update_ignore: @echo '# mypy: disable-error-code="name-defined"' >> .dev_config/.vulture_ignore.py -@poetry run vulture --make-whitelist >> .dev_config/.vulture_ignore.py || true # blocklint: pragma -.PHONY: pyupgrade -pyupgrade: - @poetry run pyupgrade $(shell git ls-files '*.py' '*.ipynb') - .PHONY: pymend pymend: @poetry run pymend $(shell git ls-files '*.py' '*.ipynb') @@ -173,15 +175,20 @@ lintprose: # prettier ~ .PHONY: find_noqa_comments find_noqa_comments: - @grep -rIn 'isort:\|noqa\|type:\|pylint\|nosec' $(shell git ls-files '*.py' '*.ipynb') || true - @grep -rIn 'yamllint' $(shell git ls-files '*.yaml' '*.yml') || true - @grep -rIn 'pragma\|blocklint:' $(shell git ls-files) || true - @grep -rIn 'markdownlint-' $(shell git ls-files '*.md') || true - @grep -rIn 'eslint' $(shell git ls-files '*.js') || true - @grep -rIn 'prettier-ignore' $(shell git ls-files '*.html' '*.scss' '*.css') || true + @grep -rInH 'isort:\|noqa\|type:\|pylint\|nosec' $(shell git ls-files '*.py' '*.ipynb') || true + @grep -rInH 'yamllint' $(shell git ls-files '*.yaml' '*.yml') || true + @grep -rInH 'pragma\|blocklint:' $(shell git ls-files) || true + @grep -rInH 'markdownlint-' $(shell git ls-files '*.md') || true + @grep -rInH 'eslint' $(shell git ls-files '*.js') || true + @grep -rInH 'prettier-ignore' $(shell git ls-files '*.html' '*.scss' '*.css') || true # Find double spaces that are not leading, and that are not before a `#` character, # i.e. indents and `code # comment` are fine, but `code # comment with extra space` is not .PHONY: find_double_spaces find_double_spaces: - @grep -rInE '[^ \n] {2,}[^#]' $(shell git ls-files ':!:poetry.lock' ':!:media' ':!:daq/logs') || true + @grep -rInHE '[^ \n] {2,}[^#]' $(shell git ls-files ':!:poetry.lock' ':!:media' ':!:daq/logs') || true + +# Find trailing spaces +.PHONY: find_trailing_spaces +find_trailing_spaces: + @grep -rInHE ' $$' $(shell git ls-files ':!:poetry.lock' ':!:media') || true diff --git a/README.md b/README.md index a083431e..8f9b77a5 100644 --- a/README.md +++ b/README.md @@ -320,7 +320,6 @@ sudo ln -s /usr/local/bin/python3.11 /usr/bin/python3 # Should match /usr/local/bin/python3.11 -VV python -VV ``` - #### Additional DAQ Dependencies To finish setting up the DAQ system you must also: @@ -331,6 +330,7 @@ To finish setting up the DAQ system you must also: `pigpio` is necessary to interface with the GPIO ports and must also be enabled via a [daemon](https://gpiozero.readthedocs.io/en/latest/remote_gpio.html) * Enable SPI, I2C, and Remote GPIO via `raspi-config`. * Prevent the [WiFi from powering off](https://desertbot.io/blog/headless-raspberry-pi-4-ssh-wifi-setup-64-bit-mac-windows). +* It is recommended to [install](https://pimylifeup.com/raspberry-pi-log2ram) [`log2ram`](https://github.com/azlux/log2ram) to avoid unnecessary writes to the SD card, prolonging the card's lifetime. ```bash # Install tmux and pigpio @@ -345,7 +345,14 @@ sudo systemctl enable pigpiod # Prevent the WiFi from powering off # Above the line that says exit 0 insert `/sbin/iw wlan0 set power_save off` and save the file sudo vi /etc/rc.local + +# Install log2ram +echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ bullseye main" | sudo tee /etc/apt/sources.list.d/azlux.list +sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg +sudo apt update && sudo apt full-upgrade -y +sudo apt install -y log2ram ``` + ### Installing Dependencies with Poetry Install `poetry` following the [instructions here](https://python-poetry.org/docs#installation). diff --git a/pyproject.toml b/pyproject.toml index 1d51574b..346aa8f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ optional = true [tool.poetry.group.dev.dependencies] pre-commit = "^3.6.2" isort = "^5.13.2" +pyupgrade = "^3.15.1" black = { extras = ["jupyter"], version = "^24.3.0" } blacken-docs = "^1.16.0" flake8 = "^7.0.0" @@ -33,7 +34,6 @@ detect-secrets = "^1.4.0" vulture = "^2.11" yamllint = "^1.35.1" blocklint = "^0.2.4" -pyupgrade = "^3.15.1" pymend = { git = "https://github.com/JanEricNitschke/pymend.git" } deptry = "^0.12.0" html5validator = "^0.4.2"