Skip to content

Commit

Permalink
Infra improvements (#69)
Browse files Browse the repository at this point in the history
* Specify python3.11 in setup

* Run all tests even if prior tests fail

* Reorder pyupgrade after isort

* Add log2ram to README

* Add trailing-spaces check

* find trailing spaces in Makefile

* Always print filenames via H

* Fix double space
  • Loading branch information
mepland authored May 21, 2024
1 parent 7293794 commit a28c32a
Show file tree
Hide file tree
Showing 5 changed files with 72 additions and 39 deletions.
59 changes: 39 additions & 20 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ jobs:
- added|modified: '**.tex'
rst:
- added|modified: '**.rst'
any:
- added|modified: '**'
- name: Free Disk Space
uses: jlumbroso/free-disk-space@main
Expand Down Expand Up @@ -121,88 +123,91 @@ jobs:

# All files
- name: detect-secrets
if: success() || failure()
run: poetry run detect-secrets-hook --exclude-lines 'integrity='

- name: blocklint
if: success() || failure()
run: poetry run blocklint --skip-files=poetry.lock --max-issue-threshold=1

# poetry
- name: poetry
if: steps.filter.outputs.poetry == 'true'
if: (success() || failure()) && steps.filter.outputs.poetry == 'true'
run: poetry check && poetry lock --no-update

# python
- name: isort
if: steps.filter.outputs.python == 'true'
if: (success() || failure()) && steps.filter.outputs.python == 'true'
run: poetry run isort ${{ steps.filter.outputs.python_files }}

- name: pyupgrade
if: (success() || failure()) && steps.filter.outputs.python == 'true'
run: poetry run pyupgrade ${{ steps.filter.outputs.python_files }}

- name: black
if: steps.filter.outputs.python == 'true'
if: (success() || failure()) && steps.filter.outputs.python == 'true'
run: poetry run black --check --diff ${{ steps.filter.outputs.python_files }}

- name: flake8
if: steps.filter.outputs.python == 'true'
if: (success() || failure()) && steps.filter.outputs.python == 'true'
run: |
echo "::group::version and plugins"
poetry run flake8 --version
echo "::endgroup::"
poetry run flake8 ${{ steps.filter.outputs.python_files }}
- name: mypy
if: steps.filter.outputs.python == 'true'
if: (success() || failure()) && steps.filter.outputs.python == 'true'
run: poetry run mypy ${{ steps.filter.outputs.python_files }}

- name: pylint
if: steps.filter.outputs.python_alone == 'true'
if: (success() || failure()) && steps.filter.outputs.python_alone == 'true'
run: poetry run pylint ${{ steps.filter.outputs.python_alone_files }}

- name: bandit
if: steps.filter.outputs.python == 'true'
if: (success() || failure()) && steps.filter.outputs.python == 'true'
run: poetry run bandit -r ${{ steps.filter.outputs.python_files }}

- name: vulture
if: steps.filter.outputs.python == 'true'
if: (success() || failure()) && steps.filter.outputs.python == 'true'
run: poetry run vulture . # Needs to run on whole package

- name: pyupgrade
if: steps.filter.outputs.python == 'true'
run: poetry run pyupgrade ${{ steps.filter.outputs.python_files }}

- name: deptry
if: steps.filter.outputs.python == 'true'
if: (success() || failure()) && steps.filter.outputs.python == 'true'
run: poetry run deptry . # Runs on whole package

# javascript
- name: standard
if: steps.filter.outputs.javascript == 'true'
if: (success() || failure()) && steps.filter.outputs.javascript == 'true'
run: standard ${{ steps.filter.outputs.javascript_files }}

# html
- name: html5validator
if: steps.filter.outputs.html == 'true'
if: (success() || failure()) && steps.filter.outputs.html == 'true'
uses: Cyb3r-Jak3/html5validator-action@master # blocklint: pragma
with:
config: .dev_config/.html5validator.yaml
# only configurable to run on paths, so just run on whole package

# prettier: html css scss json
- name: prettier
if: steps.filter.outputs.prettier == 'true'
if: (success() || failure()) && steps.filter.outputs.prettier == 'true'
id: prettier
run: prettier --ignore-path .dev_config/.prettierignore --ignore-path .gitignore --config .dev_config/.prettierrc.yaml --check ${{ steps.filter.outputs.prettier_files }} # yamllint disable-line rule:line-length

# yaml
- name: yamllint
if: steps.filter.outputs.yaml == 'true'
if: (success() || failure()) && steps.filter.outputs.yaml == 'true'
run: poetry run yamllint -c .dev_config/.yamllint.yaml --strict ${{ steps.filter.outputs.yaml_files }} # yamllint disable-line rule:line-length

# makefile
- name: checkmake
if: steps.filter.outputs.makefile == 'true'
if: (success() || failure()) && steps.filter.outputs.makefile == 'true'
uses: Uno-Takashi/checkmake-action@main # Can not pass multiple files

# shell files - no extension to filter on, always run
- name: shellcheck shfmt checkbashisms
if: success() || failure()
uses: luizm/action-sh-checker@master # blocklint: pragma
env:
SHFMT_OPTS: -bn -ci -sr -s -d
Expand All @@ -211,13 +216,14 @@ jobs:

# typos - always run
- name: typos
if: success() || failure()
uses: crate-ci/[email protected]
with:
config: pyproject.toml

# docs
- name: markdownlint
if: steps.filter.outputs.markdown == 'true'
if: (success() || failure()) && steps.filter.outputs.markdown == 'true'
uses: nosborn/[email protected]
with:
files: ${{ steps.filter.outputs.markdown_files }}
Expand All @@ -226,5 +232,18 @@ jobs:
ignore_files: LICENSE.md

- name: blacken-docs
if: steps.filter.outputs.python == 'true' || steps.filter.outputs.markdown == 'true' || steps.filter.outputs.rst == 'true' || steps.filter.outputs.tex == 'true' # yamllint disable-line rule:line-length
if: (success() || failure()) && (steps.filter.outputs.python == 'true' || steps.filter.outputs.markdown == 'true' || steps.filter.outputs.rst == 'true' || steps.filter.outputs.tex == 'true') # yamllint disable-line rule:line-length
run: poetry run blacken-docs --line-length=100 ${{ steps.filter.outputs.python_files }} ${{ steps.filter.outputs.markdown_files }} ${{ steps.filter.outputs.rst_files }} ${{ steps.filter.outputs.tex_files }} # yamllint disable-line rule:line-length

- name: trailing-spaces
if: (success() || failure()) && steps.filter.outputs.any == 'true'
run: |
TRAILING_SPACES=$(grep -rInH ' $' ${{ steps.filter.outputs.any_files }} || true)
if [[ $TRAILING_SPACES ]]; then
echo "Trailing spaces found, please remove them:"
echo "$TRAILING_SPACES"
false
else
echo "No trailing spaces found."
true
fi
12 changes: 6 additions & 6 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ repos:
hooks:
- id: isort

# pyupgrade
- repo: https://github.com/asottile/pyupgrade
rev: 'v3.15.1'
hooks:
- id: pyupgrade

# black
- repo: https://github.com/psf/black
rev: '24.3.0'
Expand Down Expand Up @@ -115,12 +121,6 @@ repos:
hooks:
- id: vulture

# pyupgrade
- repo: https://github.com/asottile/pyupgrade
rev: 'v3.15.1'
hooks:
- id: pyupgrade

# deptry - only works when run within poetry env, but can't ensure from within pre-commit
# - repo: https://github.com/fpgmaas/deptry.git
# rev: "0.12.0"
Expand Down
29 changes: 18 additions & 11 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@

.PHONY: setupDAQ
setupDAQ:
@poetry env use python3.11
@poetry install --with daq,web,dev --no-root
@poetry run pre-commit install

.PHONY: setupANA
setupANA:
@poetry env use python3.11
@poetry install --with ana,dev --no-root
@poetry run pre-commit install

Expand All @@ -31,6 +33,10 @@ pre-commit-update:
isort:
@poetry run isort .

.PHONY: pyupgrade
pyupgrade:
@poetry run pyupgrade $(shell git ls-files '*.py' '*.ipynb')

.PHONY: black
black:
@poetry run black .
Expand Down Expand Up @@ -80,10 +86,6 @@ vulture-update_ignore:
@echo '# mypy: disable-error-code="name-defined"' >> .dev_config/.vulture_ignore.py
-@poetry run vulture --make-whitelist >> .dev_config/.vulture_ignore.py || true # blocklint: pragma

.PHONY: pyupgrade
pyupgrade:
@poetry run pyupgrade $(shell git ls-files '*.py' '*.ipynb')

.PHONY: pymend
pymend:
@poetry run pymend $(shell git ls-files '*.py' '*.ipynb')
Expand Down Expand Up @@ -173,15 +175,20 @@ lintprose:
# prettier ~ <!-- prettier-ignore -->
.PHONY: find_noqa_comments
find_noqa_comments:
@grep -rIn 'isort:\|noqa\|type:\|pylint\|nosec' $(shell git ls-files '*.py' '*.ipynb') || true
@grep -rIn 'yamllint' $(shell git ls-files '*.yaml' '*.yml') || true
@grep -rIn 'pragma\|blocklint:' $(shell git ls-files) || true
@grep -rIn 'markdownlint-' $(shell git ls-files '*.md') || true
@grep -rIn 'eslint' $(shell git ls-files '*.js') || true
@grep -rIn 'prettier-ignore' $(shell git ls-files '*.html' '*.scss' '*.css') || true
@grep -rInH 'isort:\|noqa\|type:\|pylint\|nosec' $(shell git ls-files '*.py' '*.ipynb') || true
@grep -rInH 'yamllint' $(shell git ls-files '*.yaml' '*.yml') || true
@grep -rInH 'pragma\|blocklint:' $(shell git ls-files) || true
@grep -rInH 'markdownlint-' $(shell git ls-files '*.md') || true
@grep -rInH 'eslint' $(shell git ls-files '*.js') || true
@grep -rInH 'prettier-ignore' $(shell git ls-files '*.html' '*.scss' '*.css') || true

# Find double spaces that are not leading, and that are not before a `#` character,
# i.e. indents and `code # comment` are fine, but `code # comment with extra space` is not
.PHONY: find_double_spaces
find_double_spaces:
@grep -rInE '[^ \n] {2,}[^#]' $(shell git ls-files ':!:poetry.lock' ':!:media' ':!:daq/logs') || true
@grep -rInHE '[^ \n] {2,}[^#]' $(shell git ls-files ':!:poetry.lock' ':!:media' ':!:daq/logs') || true

# Find trailing spaces
.PHONY: find_trailing_spaces
find_trailing_spaces:
@grep -rInHE ' $$' $(shell git ls-files ':!:poetry.lock' ':!:media') || true
9 changes: 8 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,6 @@ sudo ln -s /usr/local/bin/python3.11 /usr/bin/python3
# Should match /usr/local/bin/python3.11 -VV
python -VV
```
<!-- markdownlint-enable MD013 -->

#### Additional DAQ Dependencies
To finish setting up the DAQ system you must also:
Expand All @@ -331,6 +330,7 @@ To finish setting up the DAQ system you must also:
`pigpio` is necessary to interface with the GPIO ports and must also be enabled via a [daemon](https://gpiozero.readthedocs.io/en/latest/remote_gpio.html)
* Enable SPI, I2C, and Remote GPIO via `raspi-config`.
* Prevent the [WiFi from powering off](https://desertbot.io/blog/headless-raspberry-pi-4-ssh-wifi-setup-64-bit-mac-windows).
* It is recommended to [install](https://pimylifeup.com/raspberry-pi-log2ram) [`log2ram`](https://github.com/azlux/log2ram) to avoid unnecessary writes to the SD card, prolonging the card's lifetime.

```bash
# Install tmux and pigpio
Expand All @@ -345,7 +345,14 @@ sudo systemctl enable pigpiod
# Prevent the WiFi from powering off
# Above the line that says exit 0 insert `/sbin/iw wlan0 set power_save off` and save the file
sudo vi /etc/rc.local

# Install log2ram
echo "deb [signed-by=/usr/share/keyrings/azlux-archive-keyring.gpg] http://packages.azlux.fr/debian/ bullseye main" | sudo tee /etc/apt/sources.list.d/azlux.list
sudo wget -O /usr/share/keyrings/azlux-archive-keyring.gpg https://azlux.fr/repo.gpg
sudo apt update && sudo apt full-upgrade -y
sudo apt install -y log2ram
```
<!-- markdownlint-enable MD013 -->

### Installing Dependencies with Poetry
Install `poetry` following the [instructions here](https://python-poetry.org/docs#installation).
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ optional = true
[tool.poetry.group.dev.dependencies]
pre-commit = "^3.6.2"
isort = "^5.13.2"
pyupgrade = "^3.15.1"
black = { extras = ["jupyter"], version = "^24.3.0" }
blacken-docs = "^1.16.0"
flake8 = "^7.0.0"
Expand All @@ -33,7 +34,6 @@ detect-secrets = "^1.4.0"
vulture = "^2.11"
yamllint = "^1.35.1"
blocklint = "^0.2.4"
pyupgrade = "^3.15.1"
pymend = { git = "https://github.com/JanEricNitschke/pymend.git" }
deptry = "^0.12.0"
html5validator = "^0.4.2"
Expand Down

0 comments on commit a28c32a

Please sign in to comment.