Make changes to support HitL testing. (#69)

.github/workflows/hil-circuitpython.yml

@@ -0,0 +1,142 @@
+#
+
+name: HIL-circuitpython
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      # This is quite a big job so run only when files affecting it change.
+      - .github/workflows/hil-circuitpython.yml
+      - examples/notecard-basics/cpy_example.py
+      - test/hitl/**
+      - test/scripts/usbmount
+      - test/scripts/check_cpy*.*
+      - notecard/**
+
+  workflow_dispatch:
+    inputs:
+      flash_device:
+        required: false
+        type: boolean
+        default: true
+
+jobs:
+  test:
+    runs-on: [self-hosted, linux, circuitpython, swan-3.0, notecard-serial]
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        CIRCUITPYTHON_VERSION: [8.2.2]
+        flash_device: # has to be an array - use the input from workflow_dispatch if present, otherwlse true
+            - ${{ github.event.inputs.flash_device=='' && true || github.event.inputs.flash_device }}
+        lock_cpy_filesystem: [true]
+    env:
+      USB_MSD_ATTACH_TIME: 15
+      CIRCUITPYTHON_UF2: "adafruit-circuitpython-swan_r5-en_US-${{ matrix.CIRCUITPYTHON_VERSION }}.uf2"
+      CIRCUITPYTHON_VERSION: ${{ matrix.CIRCUITPYTHON_VERSION}}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Set Env Vars
+        run: |
+          # environment variables set in a step cannot be used until subsequent steps
+          echo "CIRCUITPYTHON_UF2_URL=https://downloads.circuitpython.org/bin/swan_r5/en_US/${CIRCUITPYTHON_UF2}" >> $GITHUB_ENV
+
+      - name: Check Runner Config
+        run: test/scripts/check_cpy_runner_config.sh
+
+      - name: Download Latest Bootloader
+        env:
+          REPO: adafruit/tinyuf2
+          ASSET: tinyuf2-swan_r5
+        if: ${{ matrix.flash_device }}
+        run: |
+          echo "retrieving the latest release from ${REPO}"
+          wget -q -O latest.json "https://api.github.com/repos/${REPO}/releases/latest"
+
+          echo "extracting asset details for ${ASSET}"
+          asset_file="${ASSET}_asset.json"
+          jq -r --arg ASSET "$ASSET" '.assets[] | select(.name | startswith($ASSET))' latest.json > $asset_file
+
+          # extract the name and download url without double quotes
+          download_name=$(jq -r '.name' $asset_file)
+          download_url=$(jq -r '.browser_download_url' $asset_file)
+          echo "Downloading release from $download_url"
+          wget -q -N $download_url
+          unzip -o $download_name
+          binfile=$(basename $download_name .zip).bin
+          echo "TINYUF2_BIN=$binfile" >> $GITHUB_ENV
+
+      - name: Download CircuitPython v${{ env.CIRCUITPYTHON_VERSION }}
+        if: ${{ matrix.flash_device }}
+        run: |
+          echo "Downloading CircuitPython for Swan from $CIRCUITPYTHON_UF2_URL"
+          wget -q -N "$CIRCUITPYTHON_UF2_URL"
+
+      - name: Erase device and program bootloader
+        if: ${{ matrix.flash_device }}
+        run: |
+          # cannot use st-flash - every 2nd programing incorrectly puts the device in DFU mode
+          # st-flash --reset  write $binfile 0x8000000
+          # Have to use the version of openocd bundled with the STM32 platform in PlatformIO, which (presumably) has the stm32 extensions compiled in
+          ~/.platformio/packages/tool-openocd/bin/openocd \
+             -d2 -s ~/.platformio/packages/tool-openocd/openocd/scripts \
+             -f interface/stlink.cfg -c "transport select hla_swd" -f target/stm32l4x.cfg \
+             -c "init; halt; stm32l4x mass_erase 0" \
+             -c "program $TINYUF2_BIN 0x8000000 verify reset; shutdown"
+
+      - name: Program CircuitPython
+        if: ${{ matrix.flash_device }}
+        run: |
+          # wait for the bootloader drive to appear
+          timeout $USB_MSD_ATTACH_TIME bash test/scripts/wait_for_file.sh "$CPY_FS_UF2"
+
+          # The bootloader reboots quickly once the whole file has been received,
+          # causing an input/output error to be reported.
+          # Ignore that, and fail if the CIRCUITPY filesystem doesn't appear
+          echo "Uploading CircuitPython binary..."
+          cp "$CIRCUITPYTHON_UF2" "$CPY_FS_UF2" || true
+          echo Ignore the input/output error above. Waiting for device to boot.
+          timeout $USB_MSD_ATTACH_TIME bash test/scripts/wait_for_file.sh "$CPY_FS_CIRCUITPY"
+          echo "CircuitPython binary uploaded and running."
+
+      - name: Make CircuitPython filesystem writeable to pyboard
+        if: ${{ matrix.lock_cpy_filesystem }}
+        run: |
+          timeout $USB_MSD_ATTACH_TIME bash test/scripts/wait_for_file.sh "$CPY_FS_CIRCUITPY"
+
+          # only copy if it's changed or not present. After the device has reset, no further changes can be made
+          # until the filesystem is erased. This allows the workflow to be rerun flash_device=false
+          diff test/hitl/boot.py "$CPY_FS_CIRCUITPY/boot.py" || cp test/hitl/boot.py "$CPY_FS_CIRCUITPY"
+
+          # reset the device (todo move this blob to a utility script)
+          ~/.platformio/packages/tool-openocd/bin/openocd \
+          -d2 -s ~/.platformio/packages/tool-openocd/openocd/scripts \
+          -f interface/stlink.cfg -c "transport select hla_swd" -f target/stm32l4x.cfg \
+          -c "init; halt; reset; shutdown"
+
+          # wait for the device to come back
+          timeout $USB_MSD_ATTACH_TIME bash test/scripts/wait_for_file.sh "$CPY_FS_CIRCUITPY"
+
+      - name: Setup Python
+        run: |
+           python3 -m venv .venv-runner
+           . .venv-runner/bin/activate
+           pip install -r test/hitl/requirements.txt
+
+      - name: Setup 'note-python' on device
+        if: ${{ ! matrix.lock_cpy_filesystem }}
+        run: |
+          mkdir -p ${CPY_FS_CIRCUITPY}/lib/notecard
+          cp notecard/*.py ${CPY_FS_CIRCUITPY}/lib/notecard/
+          cp examples/notecard-basics/cpy_example.py ${CPY_FS_CIRCUITPY}/example.py
+
+      - name: Run CircuitPython Tests
+        run: |
+          . .venv-runner/bin/activate
+          ${{ ! matrix.lock_cpy_filesystem }} && skipsetup=--skipsetup
+          pytest $skipsetup "--productuid=$CPY_PRODUCT_UID" "--port=$CPY_SERIAL" --platform=circuitpython test/hitl

.github/workflows/hil-micropython.yml

@@ -0,0 +1,88 @@
+name: HIL-micropython
+
+on:
+  pull_request:
+    branches: [ main ]
+    paths:
+      - .github/workflows/hil-micropython.yml
+      - test/hitl/**
+      - notecard/**
+      - examples/notecard-basics/mpy_example.py
+      - test/scripts/check_mpy*.*
+
+  workflow_dispatch:
+    inputs:
+      flash_device:
+        required: false
+        type: boolean
+        default: true
+
+jobs:
+  test:
+    runs-on:
+      - self-hosted
+      - linux
+      - ${{ matrix.MPY_BOARD }}
+      - notecard-serial
+      - micropython
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        MICROPYTHON_VERSION: [1.20.0]
+        MICROPYTHON_DATE: [20230426]
+        MICROPYTHON_MCU: [esp32]
+        MPY_BOARD: [huzzah32]   # the --mpyboard parameter to the tests
+        flash_device: # has to be an array - use the input from workflow_dispatch if present, otherwlse true
+            - ${{ github.event.inputs.flash_device=='' && true || github.event.inputs.flash_device }}
+    env:
+      VENV: .venv-runner-mpy
+      USB_MSD_ATTACH_TIME: 15
+      MICROPYTHON_BIN: "${{matrix.MICROPYTHON_MCU}}-${{matrix.MICROPYTHON_DATE}}-v${{matrix.MICROPYTHON_VERSION}}.bin"
+      MICROPYTHON_VERSION: ${{matrix.MICROPYTHON_VERSION}}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Set Env Vars
+        run: |
+          # environment variables set in a step cannot be used until subsequent steps
+          echo "MICROPYTHON_BIN_URL=https://micropython.org/resources/firmware/${{env.MICROPYTHON_BIN}}" >> $GITHUB_ENV
+
+      - name: Check Runner Config
+        run: test/scripts/check_mpy_runner_config.sh
+
+      - name: Download MicroPython v${{ env.MICROPYTHON_VERSION }}
+        if: ${{ matrix.flash_device }}
+        run: |
+          echo "Downloading MicroPython for ESP32 from $MICROPYTHON_BIN_URL"
+          wget -q -N "$MICROPYTHON_BIN_URL"
+
+      - name: Setup Python
+        run: |
+           python3 -m venv ${{ env.VENV }}
+           . ${{ env.VENV }}/bin/activate
+           # esptool installed directly because it's only a dependency of this workflow
+           # while requirements.txt are dependencies of the tests in test/hitl
+           pip install -r test/hitl/requirements.txt esptool
+
+      - name: Erase device and Program Micropython
+        if: ${{ matrix.flash_device }}
+        run: |
+          . ${{ env.VENV }}/bin/activate
+          # esptool requires the flash to be erased first
+          esptool.py --chip esp32 -p ${MPY_SERIAL} erase_flash
+          timeout 10 bash test/scripts/wait_for_file.sh "$MPY_SERIAL"
+
+          esptool.py --chip esp32 --port ${MPY_SERIAL} --baud 460800 write_flash -z 0x1000 ${{ env.MICROPYTHON_BIN }}
+          timeout 10 bash test/scripts/wait_for_file.sh "$MPY_SERIAL"
+
+          # wait for MicroPython to complete initial setup
+          echo "help()" >> "$MPY_SERIAL"
+          sleep 10
+
+      - name: Run MicroPython Tests
+        run: |
+          . ${{ env.VENV }}/bin/activate
+          pytest "--productuid=$MPY_PRODUCT_UID" "--port=$MPY_SERIAL" --platform=micropython --mpyboard=${{ matrix.MPY_BOARD }} test/hitl

.github/workflows/python-ci.yml

@@ -78,7 +78,7 @@ jobs:
           DD_SERVICE: note-python
           DD_ENV: ci
         run: |
-          coverage run -m pytest --ddtrace --ddtrace-patch-all
+          coverage run -m pytest --ddtrace --ddtrace-patch-all --ignore=test/hitl
       - name: Publish to Coveralls
         env:
             GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+
+-   repo: local
+    hooks:
+    -   id: Formatting
+        name: Formatting
+        entry: make precommit
+        language: python    # This sets up a virtual environment
+        additional_dependencies: [flake8, pydocstyle]

Makefile

@@ -1,40 +1,46 @@
 # define VENV_NAME to use a specific virtual environment. It defaults to `env`.
 VENV_NAME?=env
-VENV_ACTIVATE=. $(VENV_NAME)/bin/activate
+VENV_ACTIVATE=$(VENV_NAME)/bin/activate
 PYTHON=python
-VENV =
+# the target to activate the virtual environment. Only defined if it exists.
 
-# check if the VENV file exists
-ifneq ("$(wildcard $(PVENV_ACTIVATE))","")
-    VENV = venv
-    PYTHON = ${VENV_NAME}/bin/python3
+# check if the VENV file exists, if it does assume that's been made active
+ifneq ("$(wildcard ${VENV_ACTIVATE})","")
+	RUN_VENV_ACTIVATE=. ${VENV_ACTIVATE}
+	PYTHON = ${VENV_NAME}/bin/python3
 endif
 
-default: docstyle flake8 test
+default: precommit
 
-venv: $(VENV_NAME)/bin/activate
+precommit: docstyle flake8
 
-test: $(VENV)
-	${PYTHON} -m pytest test --cov=notecard
+test:
+	${RUN_VENV_ACTIVATE}
+	${PYTHON} -m pytest test --cov=notecard --ignore=test/hitl
 
-docstyle: $(VENV)
-	${PYTHON} -m pydocstyle notecard/ examples/
+docstyle:
+	${RUN_VENV_ACTIVATE}
+	${PYTHON} -m pydocstyle notecard/ examples/ mpy_board/
 
-flake8: $(VENV)
-    # E722 Do not use bare except, specify exception instead https://www.flake8rules.com/rules/E722.html
-    # F401 Module imported but unused   https://www.flake8rules.com/rules/F401.html
-    # F403 'from module import *' used; unable to detect undefined names  https://www.flake8rules.com/rules/F403.html
-    # W503 Line break occurred before a binary operator https://www.flake8rules.com/rules/W503.html
-    # E501 Line too long (>79 characters) https://www.flake8rules.com/rules/E501.html
-	${PYTHON} -m flake8 test/ notecard/ examples/ --count --ignore=E722,F401,F403,W503,E501 --show-source --statistics
+flake8:
+	${RUN_VENV_ACTIVATE}
+	# E722 Do not use bare except, specify exception instead https://www.flake8rules.com/rules/E722.html
+	# F401 Module imported but unused   https://www.flake8rules.com/rules/F401.html
+	# F403 'from module import *' used; unable to detect undefined names  https://www.flake8rules.com/rules/F403.html
+	# W503 Line break occurred before a binary operator https://www.flake8rules.com/rules/W503.html
+	# E501 Line too long (>79 characters) https://www.flake8rules.com/rules/E501.html
+	${PYTHON} -m flake8 test/ notecard/ examples/ mpy_board/ --count --ignore=E722,F401,F403,W503,E501 --show-source --statistics
 
-coverage: $(VENV)
-	${PYTHON} -m pytest test --doctest-modules --junitxml=junit/test-results.xml --cov=notecard --cov-report=xml --cov-report=html
+coverage:
+	${RUN_VENV_ACTIVATE}
+	${PYTHON} -m pytest test --ignore=test/hitl --doctest-modules --junitxml=junit/test-results.xml --cov=notecard --cov-report=xml --cov-report=html
 
-run_build: $(VENV)
+run_build:
+	${RUN_VENV_ACTIVATE}
 	${PYTHON} -m setup sdist bdist_wheel
 
-deploy: $(VENV)
+deploy:
+	${RUN_VENV_ACTIVATE}
 	${PYTHON} -m twine upload -r "pypi" --config-file .pypirc 'dist/*'
 
-.PHONY: venv test coverage run_build deploy
+.PHONY: precommit venv test coverage run_build deploy

README.md

@@ -136,11 +136,11 @@ The documentation for this library can be found
 The [examples](examples/) directory contains examples for using this
 library with:
 
-- [Serial](examples/notecard-basics/serial-example.py)
-- [I2C](examples/notecard-basics/i2c-example.py)
-- [RaspberryPi](examples/notecard-basics/rpi-example.py)
-- [CircuitPython](examples/notecard-basics/cpy-example.py)
-- [MicroPython](examples/notecard-basics/mpy-example.py)
+- [Serial](examples/notecard-basics/serial_example.py)
+- [I2C](examples/notecard-basics/i2c_example.py)
+- [RaspberryPi](examples/notecard-basics/rpi_example.py)
+- [CircuitPython](examples/notecard-basics/cpy_example.py)
+- [MicroPython](examples/notecard-basics/mpy_example.py)
 
 ## Contributing
 
@@ -170,6 +170,21 @@ Alternatively, you can inspect the contents of the [Makefile](Makefile) and run
 `Makefile` run against every pull request, so your best bet is to ensure these
 tests are successful before submitting your PR.
 
+## Installing the `pre-commit` Hook
+
+Please run
+
+`pre-commit install`
+
+Before committing to this repo. It will catch a lot of common errors that you can fix locally.
+
+You may also run the pre-commit checks before committing with
+
+`pre-commit run`
+
+Note that `pre-commit run` only considers staged changes, so be sure all
+changes are staged before running this.
+
 ## More Information
 
 For additional Notecard SDKs and Libraries, see:

examples/notecard-basics/board.py

@@ -0,0 +1,21 @@
+"""
+Define peripherals for different types of boards.
+
+This module, or it's variants are  used by the mpy_example to use the appropriate
+UART or I2C configuration for the particular board being used.
+The values here are defaults. The definitions for real boards are located in ./mpy_board/*
+at the root of the repo.
+"""
+
+
+"""
+The UART instance to use that is connected to Notecard.
+"""
+UART = 2
+
+"""
+The I2C ID and SDL and SDA pins of the I2C bus connected to Notecard
+"""
+I2C_ID = 0
+SCL = 0
+SDA = 0