Refactor card.binary HIL GitHub workflow. (#144)

- Remove the unused localtunnel code. If we ever need to switch to localtunne from tunnelmole, we can pull it out of the git history. - Remove usage of third party action pyTooling/Actions/with-post-step. Instead, to run cleanup steps, use the `if: always()` construct. - Remove usage of third party action JarvusInnovations/background-action@v1. The scripts we need to run in the background can simply be launched in the background using & syntax. - Refactor a bunch of conditional logic that was hard to read. Instead of using if statements, we were chaining together a bunch of different Bash statements using logical operators. Using if statements is simpler to read and understand what exactly is being checked. - Refactor run_tunnelmole.sh.
blues · Mar 25, 2024 · a4f8c65 · a4f8c65
1 parent bd9556e
commit a4f8c65
Show file tree

Hide file tree

Showing 3 changed files with 125 additions and 153 deletions.
diff --git a/.github/workflows/notecard-binary-tests.yml b/.github/workflows/notecard-binary-tests.yml
@@ -41,39 +41,24 @@ jobs:
  # START_MD5SRV set to false to skip starting the MD5 server. There should be one
  # already running locally with MD5SRV_PORT/ADDRESS/TOKEN set correspondingly.
  START_MD5SRV: true
- # START_LOCALTUNNEL, set to false to skip starting the localtunnel.
- START_LOCALTUNNEL: false
  # START_TUNNELMOLE: set to false to skip starting tunnel mole.
  START_TUNNELMOLE: true
  # When neither tunneling solution is used (because they're already instantiated outside of the workflow)
  # be sure to set MD5SRV_URL in the environment
  steps:
- - name: Checkout
+ - name: Checkout note-c repo
  uses: actions/checkout@v3
+
  - name: Generate MD5 Server Token
  run: |
  [ -n "$MD5SRV_TOKEN" ] || echo "MD5SRV_TOKEN=`uuidgen`" >> $GITHUB_ENV
  # propagate the environment variable so that it's available in the `env` context
  echo "MD5SRV_PORT=$MD5SRV_PORT" >> $GITHUB_ENV
- - name: Check Env Vars
+
+ - name: Check env vars
  run: |
  . scripts/check_runner_config.sh
  echo NOTEHUB_PROXY_ROUTE_ALIAS=$NOTEHUB_PROXY_ROUTE_ALIAS
- - name: Prep MD5 Server
- uses: pyTooling/Actions/[email protected]
- with:
- main: |
- [ -e md5srv-files ] || mkdir md5srv-files
- rm -rf md5srv-files/*
- md5url=http://${MD5SRV_ADDRESS}:${MD5SRV_PORT}/
- post: |
- # the JarvusInnovations/background-action@v1 that launches the background
- # process doesn't clean them up. We do that here. MD5SRV_PID is set in the
- # ./scripts/run_md5srv.sh script
- echo Stop MD5 Server
- [ -n "$MD5SRV_PID" ] || (echo "MD5SRV_PID not set" && exit 1)
- [ -z "$MD5SRV_PID" ] || kill $MD5SRV_PID
- rm -rf md5srv-files
 
  - name: Install PlatformIO dependencies
  if: env.FLASH_TEST_FIRMWARE!='false'
@@ -90,33 +75,12 @@ jobs:
  # copy only files in note-c
  find "$GITHUB_WORKSPACE" -maxdepth 1 -type f -exec cp "{}" "${NOTE_C_DEP}" \;
 
- - name: Start MD5 Server
- uses: JarvusInnovations/background-action@v1
- with:
- run: |
- ./scripts/run_md5srv.sh
- wait-on:
- # just a dummy wait-on since this is required.
- file:${{github.workspace}}/scripts/run_md5srv.sh
-
- # When done this way, the background process is terminated at the end of the step,
- # At least when running with `act`. The same may be true of github runners also.
- # - name: Start MD5 Server
- # uses: pyTooling/Actions/[email protected]
- # with:
- # main: |
- # ./run_md5srv.sh
- # echo "MD5SRV_PID=$MD5SRV_PID"
- # echo "MD5SRV_PID=$MD5SRV_PID" >> $GITHUB_ENV
- # echo "writing server log to `realpath md5srv.log`"
- # md5url=http://${MD5SRV_ADDRESS}:${MD5SRV_PORT}/
- # post: |
- # echo Stop MD5 Server
- # [ -n "$MD5SRV_PID" ] || (echo "MD5SRV_PID not set" && exit 1)
- # # [ -z "$MD5SRV_PID" ] || kill $MD5SRV_PID
- # rm -rf md5srv-files
-
- - name: Build and Upload Test Firmware
+ - name: Start MD5 server
+ run: |
+ mkdir md5srv-files
+ ./scripts/run_md5srv.sh
+
+ - name: Build and upload test firmware
  if: env.FLASH_TEST_FIRMWARE!='false'
  run: |
  source venv/bin/activate
@@ -126,118 +90,82 @@ jobs:
  platformio test -e debug --without-testing --upload-port "$STLINK_PROGRAMMER_PORT" --project-dir "$PIO_PROJECT_DIR"
  timeout 10 ./scripts/wait_for_file.sh "$SWAN_SERIAL"
 
- - name: Start localtunnel
- if: env.START_LOCALTUNNEL!='false'
- id: localtunnel
- uses: Rei-x/expose-localtunnel-action@main
- with:
- ports: ${{ env.MD5SRV_PORT }}
- - name: Fetch localtunnel URL
- if: env.START_LOCALTUNNEL!='false'
- run: |
- MD5SRV_URL="${{steps.localtunnel.outputs.url-1}}"
- echo "MD5SRV_URL=$MD5SRV_URL" >> $GITHUB_ENV
-
- - name: Prep tunnelmole
- if: env.START_TUNNELMOLE!='false'
- uses: pyTooling/Actions/[email protected]
- with:
- main: |
- [ ! -e tmole.log ] || rm tmole.log
- sleep 2 # otherwise it thinks we exited early
- post: |
- # the JarvusInnovations/background-action@v1 that launches the background
- # process doesn't clean them up. We do that here. TMOLE_PID is set in the
- # ./scripts/run_tunnelmole.sh script
- echo Stop Tunnelmole
- [ -n "$TMOLE_PID" ] || (echo "TMOLE_PID not set" && exit 1)
- [ -z "$TMOLE_PID" ] || kill $TMOLE_PID
- # Remove the tmole_ready file, which may be leftover from a prior
- # run.
- rm -f $GITHUB_WORKSPACE/tmole_ready
-
  - name: Start tunnelmole
- uses: JarvusInnovations/background-action@v1
  if: env.START_TUNNELMOLE!='false'
- with:
- run: |
- ./scripts/run_tunnelmole.sh
- log-output-if: true
- wait-on:
- file:${{github.workspace}}/tmole_ready
+ run: |
+ rm -f tmole.log
+ ./scripts/run_tunnelmole.sh
 
- - name: Check server is available
+ - name: Check MD5 server is available
  run: |
  # the request will return a 401 from md5srv, but that's expected without the access token
  # Curl still returns success because it could contact the server
  code=`curl -s -o /dev/null -w "%{http_code}" $MD5SRV_URL`
- [ "$code" -lt "500" ] || ( echo "5xx error ($code) from tunnel." && exit 1 )
+ if [ "$code" -ge "500" ]; then
+ echo "5xx error ($code) from tunnel."
+ exit 1
+ else
+ echo "MD5 server is available."
+ fi
 
  - name: Create Notehub accesss token
  if: env.CREATE_NOTEHUB_ROUTES!='false'
  run: |
-  curl -f -X POST \
-  -L 'https://${{env.NOTEHUB}}/oauth2/token' \
-  -H 'content-type: application/x-www-form-urlencoded' \
-  -d grant_type=client_credentials \
-  -d client_id=$NOTEHUB_CLIENT_ID \
-  -d client_secret=$NOTEHUB_CLIENT_SECRET | \
-  { token=$(jq -r .access_token); echo "NOTEHUB_ACCESS_TOKEN=$token" >> $GITHUB_ENV; }
-
- - name: Create Notehub HTTP Route
+ curl -f -X POST \
+ -L 'https://${{env.NOTEHUB}}/oauth2/token' \
+ -H 'content-type: application/x-www-form-urlencoded' \
+ -d grant_type=client_credentials \
+ -d client_id=$NOTEHUB_CLIENT_ID \
+ -d client_secret=$NOTEHUB_CLIENT_SECRET | \
+ { token=$(jq -r .access_token); echo "NOTEHUB_ACCESS_TOKEN=$token" >> $GITHUB_ENV; }
+
+ - name: Create Notehub HTTP route
  if: env.CREATE_NOTEHUB_ROUTES!='false'
- uses: pyTooling/Actions/[email protected]
- with:
- main: |
- # ?note=1 instructs the MD5 server to process the content as an event, extracting the path
- # from the event body.
- route_req=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_HTTP_ROUTE_LABEL" --arg URL "$MD5SRV_URL/?note=1" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
- '{ "label":$LABEL, "type":"http", "http":{ "timeout":$TIMEOUT, "filter": { "type":"include", "files": ["cardbinary.qo"] }, "url":$URL, "http_headers": { "X-Access-Token":$TOKEN } } }'`
- echo $route_req
- route=`echo "$route_req" | curl -s -f -X POST -L "https://$NOTEHUB_API/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
- -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
- echo $route
- route_uid=`echo $route | jq -r .uid`
- [ -n "$route_uid" ]
+ run: |
+ # ?note=1 instructs the MD5 server to process the content as an event, extracting the path
+ # from the event body.
+ route_req=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_HTTP_ROUTE_LABEL" --arg URL "$MD5SRV_URL/?note=1" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
+ '{ "label":$LABEL, "type":"http", "http":{ "timeout":$TIMEOUT, "filter": { "type":"include", "files": ["cardbinary.qo"] }, "url":$URL, "http_headers": { "X-Access-Token":$TOKEN } } }'`
+ echo "Notehub HTTP route request: $route_req"
+ route=`echo "$route_req" | curl -s -f -X POST -L "https://$NOTEHUB_API/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
+ -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
+ echo "Notehub HTTP route: $route"
+ route_uid=`echo $route | jq -r .uid`
+ if [ -n "$route_uid" ]; then
  echo "NOTEHUB_HTTP_ROUTE_UID=$route_uid" >> $GITHUB_ENV
- post: |
- echo Delete Notehub HTTP Route
- [ "$DELETE_NOTEHUB_ROUTES" == "false" ] || ([ -n "$NOTEHUB_HTTP_ROUTE_UID" ] && curl -f -s -X DELETE \
- -L "https://$NOTEHUB_API/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_HTTP_ROUTE_UID" \
- -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN")
+ else
+ echo "Failed to create or parse Notehub HTTP route."
+ exit 1
+ fi
 
- - name: Create Notehub Proxy Route
- if: env.CREATE_NOTEHUB_ROUTES!='false'
- uses: pyTooling/Actions/[email protected]
- with:
- main: |
- ALIAS=$NOTEHUB_PROXY_ROUTE_ALIAS
- route=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_PROXY_ROUTE_LABEL" --arg URL "$MD5SRV_URL" --arg ALIAS "$ALIAS" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
- '{ "label":$LABEL, "type":"proxy", "proxy":{ "timeout":$TIMEOUT, "url":$URL, "alias":$ALIAS, "http_headers": { "X-Access-Token":$TOKEN } } }' \
- | curl -s -f -X POST -L "https://api.notefile.net/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
- -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
- echo $route
- route_uid=`echo $route | jq -r .uid`
- [ -n $route_uid ]
+ - name: Create Notehub proxy route
+ run: |
+ ALIAS="$NOTEHUB_PROXY_ROUTE_ALIAS"
+ route=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_PROXY_ROUTE_LABEL" --arg URL "$MD5SRV_URL" --arg ALIAS "$ALIAS" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
+ '{ "label":$LABEL, "type":"proxy", "proxy":{ "timeout":$TIMEOUT, "url":$URL, "alias":$ALIAS, "http_headers": { "X-Access-Token":$TOKEN } } }' \
+ | curl -s -f -X POST -L "https://api.notefile.net/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
+ -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
+ echo "Notehub proxy route: $route"
+ route_uid=`echo "$route" | jq -r .uid`
+ if [ -n $route_uid ]; then
  echo "NOTEHUB_PROXY_ROUTE_UID=$route_uid" >> $GITHUB_ENV
  echo "NOTEHUB_PROXY_ROUTE_ALIAS=$ALIAS" >> $GITHUB_ENV
- post: |
- echo Delete Notehub Proxy Route
- [ "$DELETE_NOTEHUB_ROUTES" == "false" ] || ([ -n "$NOTEHUB_PROXY_ROUTE_UID" ] && curl -f -s -X DELETE \
- -L "https://api.notefile.net/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_PROXY_ROUTE_UID" \
- -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN")
+ else
+ echo "Failed to create or parse Notehub proxy route."
+ exit 1
+ fi
 
- - name: Run Tests
+ - name: Run tests
  run: |
  source venv/bin/activate
  cd $PIO_PROJECT_DIR
  platformio test -v -e debug \
  --without-building --without-uploading \
  --test-port "$SWAN_SERIAL" \
  --json-output-path test.json \
- --junit-output-path test.xml \
+ --junit-output-path test.xml
 
- - name: Publish Test Report
+ - name: Publish test report
  uses: mikepenz/action-junit-report@v3
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -246,3 +174,48 @@ jobs:
  report_paths: '**/test/hitl/card.binary/test*.xml'
  check_name: Notecard Binary HIL Tests
  require_tests: true
+
+ - name: Cleanup Notehub proxy route
+ if: always()
+ run: |
+ if [ "$DELETE_NOTEHUB_ROUTES" == "true" && -n "$NOTEHUB_PROXY_ROUTE_UID" ]; then
+ echo "Deleting Notehub proxy route."
+ curl -f -s -X DELETE \
+ -L "https://api.notefile.net/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_PROXY_ROUTE_UID" \
+ -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN"
+ fi
+
+ - name: Cleanup Notehub HTTP route
+ if: always()
+ run: |
+ if [ "$DELETE_NOTEHUB_ROUTES" == "true" ] && [ -n "$NOTEHUB_HTTP_ROUTE_UID" ]; then
+ echo "Deleting Notehub HTTP route."
+ curl -f -s -X DELETE \
+ -L "https://$NOTEHUB_API/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_HTTP_ROUTE_UID" \
+ -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN"
+ fi
+
+
+ - name: Cleanup tunnelmole
+ if: always()
+ run: |
+ if [ -n "$TMOLE_PID" ]; then
+ echo "Stopping tunnelmole."
+ kill $TMOLE_PID
+ else
+ echo "Tunnelmole not running (TMOLE_PID is empty)."
+ fi
+
+ - name: Cleanup MD5 server
+ if: always()
+ run: |
+ if [ -d md5srv-files ]; then
+ echo "Deleting md5srv-files directory."
+ rm -rf md5srv-files
+ fi
+ if [ -n "$MD5SRV_PID" ]; then
+ echo "Stopping MD5 server."
+ kill $MD5SRV_PID
+ else
+ echo "MD5 server not running (MD5SRV_PID is empty)."
+ fi
diff --git a/scripts/run_md5srv.sh b/scripts/run_md5srv.sh
@@ -2,6 +2,3 @@
 python3 ./test/hitl/scripts/md5srv.py --dir md5srv-files --save > md5srv.log 2>&1 &
 MD5SRV_PID=$!
 echo "MD5SRV_PID=$MD5SRV_PID" >> $GITHUB_ENV
-
-# Prevent the action from thinking we exited early
-sleep 2
diff --git a/scripts/run_tunnelmole.sh b/scripts/run_tunnelmole.sh
@@ -25,26 +25,28 @@ fi
 
 sleep 1
 
-# find the url
-exec 3< tmole.log
-# ignore the first line
-read <&3 output
-read <&3 output
-
-echo "Getting MD5 server URL from tmole.log..." >&2
-MD5SRV_URL=`echo "$output" | grep https | cut -d " " -f1`
+
+TIMEOUT=8
+SECONDS=0
+MD5SRV_URL=""
+# Check tmole.log for the MD5 server URL every second.
+until [ "$SECONDS" -ge "$TIMEOUT" ]
+do
+ MD5SRV_URL=$(grep -oP "^https://[\w\d\.-]+" tmole.log)
+
+ if [ -n "$MD5SRV_URL" ]; then
+ break
+ else
+ SECONDS=$((SECONDS+1))
+ sleep 1
+ fi
+done
+
 if [ -z "$MD5SRV_URL" ]; then
- echo "ERROR: Failed to get MD5 server URL." >&2
+ echo "ERROR: Timed out waiting for MD5 server URL to get written to tmole.log." >&2
  exit 1
-else
- echo "INFO: Got MD5 server URL." >&2
 fi
 
+echo "INFO: Got MD5 server URL from tmole.log." >&2
 echo "MD5SRV_URL=$MD5SRV_URL" >> $GITHUB_ENV
-# Only create tmole_ready once MD5SRV_URL has been set, as the next step in the
-# GitHub workflow depends on it.
-touch tmole_ready
-
-sleep 2
-
 echo "INFO: tmole ready. Logging to `realpath tmole.log`" >&2