Implement CI for 64-bit CVA6 (#452)

This PR extends our CI system to also run tests on real CVA6 hardware, to ensure that we don't inadvertently break this platform. The CI infrastructure depends on a bitstream version that includes our changes to the bootrom (openhwgroup/cva6#2267), which implement a simple UART-based firmware flashing functionality. As part of extending our CI to this platform, we spent some effort to make the system hostable and extensible by others. We check in some previously unofficial helper scripts that operate an FT245R relay board, enabling us to turn boards off and on remotely. We also include some infrastructure for discovering serial ports (since our runner machine has many of these).
keystone-enclave · Jun 24, 2024 · 029a3d1 · 029a3d1
1 parent e310f6a
commit 029a3d1
Show file tree

Hide file tree

Showing 27 changed files with 731 additions and 188 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -222,7 +222,7 @@ jobs:
   # Combine cache directories to save space
   combine-caches:
     runs-on: ubuntu-latest
-    if: ${{ always() }}
+    if: success() || failure()
     needs: build
     steps:
       - name: Install dependencies
@@ -378,6 +378,6 @@ jobs:
     uses: ./.github/workflows/build-runtime.yml
 
   # System tests, which are run for simulatable and self-hostable platforms
-  test-system:
+  test-system-functionality:
     needs: build
     uses: ./.github/workflows/test-system.yml
diff --git a/.github/workflows/test-system.yml b/.github/workflows/test-system.yml
@@ -3,19 +3,31 @@ on:
   workflow_call:
 
 jobs:
-  test-generic:
-    runs-on: ubuntu-latest
+  test-system:
+    runs-on: ${{ matrix.platform == 'generic' && 'ubuntu-latest' || matrix.platform }}
+    environment: ${{ matrix.platform != 'generic' && 'track' || null }}
     strategy:
+      fail-fast: false
       matrix:
-        platform: [generic]
+        platform: [generic, mpfs, cva6]
         bits: [32, 64]
+        exclude:
+          # mpfs is not 32 bit
+          - platform: mpfs
+            bits: 32
+          # ignore 32-bit cva6 for now
+          - platform: cva6
+            bits: 32
 
     steps:
-    # We don't need submodules here since Keystone is a monorepo!
     - name: Checkout Keystone
       uses: actions/checkout@v4
       with:
-        submodules: 'false'
+        submodules: 'true'
+        sparse-checkout: |
+          .
+          scripts/
+          mkutils/
 
     - name: Restore build directory
       uses: actions/download-artifact@v4
@@ -26,188 +38,63 @@ jobs:
     - name: Decompress build directory
       run: cat build.tar.xz | xz -d -T0 | tar -xf -
 
-    - name: Test Keystone system
-      run: |
-        # Fix permissions on the key
-        chmod 600 build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/target/root/.ssh/id-rsa
-
-        # Launch QEMU
-        export KEYSTONE_PLATFORM=${{ matrix.platform }}
-        export KEYSTONE_BITS=${{ matrix.bits }}
-        export QEMU_PORT=$(( RANDOM + 1024 ))
-        export LD_LIBRARY_PATH=build-${{ matrix.platform }}${{ matrix.bits }}/buildroot.build/host/lib
-        screen -L -dmS qemu bash -c "make run 2>&1 | tee run.log"
-
-        # TODO: check for connectivity instead of sleeping
-        sleep 60
-
-        export CALL_LOGFILE=cmd.log
-        echo "" > $CALL_LOGFILE
-
-        KEYSTONE_COMMAND="modprobe keystone-driver" make call
-        KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call
-        KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call
-        KEYSTONE_COMMAND="poweroff" make call
-
-    - name: Check expected
-      run: |
-        [[ -z $(diff cmd.log scripts/ci/expected.log) ]]
-
-    - name: Upload run log
-      if: failure()
-      uses: actions/upload-artifact@v4
-      with:
-        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-run.log
-        path: run.log
-
-    - name: Upload cmd log
-      if: failure()
-      uses: actions/upload-artifact@v4
-      with:
-        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log
-        path: cmd.log
-
-  test-mpfs:
-    runs-on: [self-hosted, mpfs]
-    environment: track
-    steps:
-    # We don't need submodules here since Keystone is a monorepo!
-    - name: Checkout Keystone
-      uses: actions/checkout@v4
-      with:
-        submodules: 'false'
-
-    - name: Restore build directory
-      uses: actions/download-artifact@v4
-      with:
-        name: keystone-mpfs64-builddir
-        path: .
-
-    - name: Decompress build directory
-      run: cat build.tar.xz | xz -d -T0 | tar -xf -
-
-    # Test the firmware, first by flashing it
-    - name: Flash HSS
+    - name: Flash and check firmware
       env:
-        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
-        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
-        SC_INSTALL_DIR: ${{ vars.SC_INSTALL_DIR }}
+        KEYSTONE_PLATFORM: ${{ matrix.platform }}
+        KEYSTONE_BITS: ${{ matrix.bits }}
+        LOGFILE: fw-program.log
       run: |
-        $POWER_ON_CMD
-        export FPGENPROG=$(which fpgenprog)
-        make -C build-mpfs64/buildroot.build/build/hss-v2023.06 program 2>/dev/null >program.log
-        $POWER_OFF_CMD
-
-        # Check if we succeeded
-        [[ ! -z $(cat program.log | grep "mpfsBootmodeProgrammer completed successfully") ]]
+        if [[ -f scripts/ci/plat/${{ matrix.platform }}/flash-firmware.sh ]]; then
+          scripts/ci/plat/${{ matrix.platform }}/flash-firmware.sh
+        fi
 
-    - name: Upload HSS program log
-      if: failure()
+    - name: Upload firmware programming log
+      if: ${{ matrix.platform != 'generic' && failure() }}
       uses: actions/upload-artifact@v4
       with:
-        name: test-keystone-mpfs64-prog-hss.log
-        path: program.log
+        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-fw-program.log
+        path: fw-program.log
 
-    # And then verifying that we can actually get to the command line
-    - name: Check HSS ok
+    - name: Flash and check OS
       env:
-        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
-        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
-        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
+        KEYSTONE_PLATFORM: ${{ matrix.platform }}
+        KEYSTONE_BITS: ${{ matrix.bits }}
+        LOGFILE: os-program.log
       run: |
-        # Collect serial output
-        TTYDEV=$($FIND_TTY_CMD 0)
-        screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run-hss.log"
-        $POWER_ON_CMD ; sleep 30 ; $POWER_OFF_CMD
-        screen -XS mpfs-tty quit
-
-        # At least the first hart should have started
-        [[ ! -z $(cat run-hss.log | sed -e 's/\x1b\[[0-9;]*m//g' | grep "u54 State Change:  \[Running\]") ]]
+        if [[ -f scripts/ci/plat/${{ matrix.platform }}/flash-os.sh ]]; then
+          scripts/ci/plat/${{ matrix.platform }}/flash-os.sh
+        fi
 
-    - name: Upload HSS run log
-      if: failure()
+    - name: Upload OS programming log
+      if: ${{ matrix.platform != 'generic' && failure() }}
       uses: actions/upload-artifact@v4
       with:
-        name: test-keystone-mpfs64-run-hss.log
-        path: run-hss.log
-
-    # Now we also need to flash the disk. First, get into usbdmsc
-    - name: Flash OS
-      env:
-        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
-        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
-        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
-      run: |
-        # Wait for the board to come up a bit. We'll hammer it with serial
-        # input to ensure that we halt the boot at HSS
-        TTYDEV=$($FIND_TTY_CMD 0)
-        $POWER_ON_CMD
-        NOW=$(date +%s)
-        stty raw -echo 115200 < "$TTYDEV"
-        while [[ $(( $(date +%s) - $NOW )) -lt 10 ]]; do echo 'a' > "$TTYDEV" ; done
-
-        echo "" > "$TTYDEV"
-        echo "usbdmsc" > "$TTYDEV"
-
-        # Wait a bit for the USB to connect then flash
-        sleep 10
-        FOUND_DEVICE=""
-        for d in /dev/sd? ; do
-          if [[ ! -z $(udevadm info --query=all -n "$d" | grep -i polarfire) ]]; then
-            FOUND_DEVICE="yes"
-            dd if=build-mpfs64/buildroot.build/images/sdcard.img of="$d" bs=4M oflag=direct
-            break
-          fi
-        done
-
-        $POWER_OFF_CMD
-        [[ ! -z "$FOUND_DEVICE" ]]
+        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-os-program.log
+        path: os-program.log
 
     - name: Test Keystone system
       env:
-        POWER_ON_CMD: ${{ vars.POWER_ON_CMD_MPFS }}
-        POWER_OFF_CMD: ${{ vars.POWER_OFF_CMD_MPFS }}
-        FIND_TTY_CMD: ${{ vars.FIND_TTY_CMD }}
-        KEYSTONE_IP: ${{ vars.BOARD_IP_MPFS }}
+        KEYSTONE_PLATFORM: ${{ matrix.platform }}
+        KEYSTONE_BITS: ${{ matrix.bits }}
+        LOGFILE: run.log
+        CMD_LOGFILE: cmd.log
       run: |
-        # Fix permissions on the key
-        chmod 600 build-mpfs64/buildroot.build/target/root/.ssh/id-rsa
-
-        # Start the board
-        TTYDEV=$($FIND_TTY_CMD 1)
-        export KEYSTONE_PLATFORM=mpfs
-        export KEYSTONE_BITS=64
-        screen -L -dmS mpfs-tty bash -c "stty raw -echo 115200 < $TTYDEV ; cat $TTYDEV > run.log"
-        $POWER_ON_CMD
-
-        # TODO: check for connectivity instead of sleeping
-        sleep 60
-
-        export CALL_LOGFILE=cmd.log
-        echo "" > $CALL_LOGFILE
-
-        KEYSTONE_COMMAND="modprobe keystone-driver" make call
-        KEYSTONE_COMMAND="/usr/share/keystone/examples/tests.ke" make call
-        # Todo: attestation does not yet work in mpfs
-        #KEYSTONE_COMMAND="/usr/share/keystone/examples/attestor.ke" make call
-
-        $POWER_OFF_CMD
-        screen -XS mpfs-tty quit
+        scripts/ci/plat/${{ matrix.platform }}/test.sh
 
     - name: Check expected
       run: |
-        [[ -z $(diff cmd.log scripts/ci/expected-mpfs.log) ]]
+        [[ -z $(diff -wB cmd.log scripts/ci/plat/${{ matrix.platform }}/expected.log) ]]
 
     - name: Upload run log
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: test-keystone-mpfs64-run.log
+        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-run.log
         path: run.log
 
     - name: Upload cmd log
       if: failure()
       uses: actions/upload-artifact@v4
       with:
-        name: test-keystone-mpfs64-cmd.log
+        name: test-keystone-${{ matrix.platform }}${{ matrix.bits }}-cmd.log
         path: cmd.log
diff --git a/.gitmodules b/.gitmodules
@@ -10,3 +10,6 @@
 [submodule "overlays/microchip"]
 	path = overlays/microchip
 	url = https://github.com/linux4microchip/buildroot-external-microchip
+[submodule "scripts/ci/utils/relay_ft245r"]
+	path = scripts/ci/utils/relay_ft245r
+	url = https://github.com/vpatron/relay_ft245r
diff --git a/mkutils/plat/cva6/run.mk b/mkutils/plat/cva6/run.mk
@@ -5,16 +5,30 @@
 
 
 PAYLOAD                 = $(BUILDROOT_BUILDDIR)/images/fw_payload.bin
-SDDEVICE_PART1          = $(SD_DEVICE)1
-SDDEVICE_PART2          = $(SD_DEVICE)2
-
-
+KERNEL			= $(BUILDROOT_BUILDDIR)/images/uImage
+SDDEVICE_PART1          = $(shell lsblk $(SD_DEVICE) -no PATH | head -2 | tail -1)
+SDDEVICE_PART2          = $(shell lsblk $(SD_DEVICE) -no PATH | head -3 | tail -1)
 
 flash: $(SD_DEVICE)
 	$(info PAYLOAD INFORMATION)
 	$(info $(PAYLOAD))
 	$(info $(SD_DEVICE))
 	$(info $(SDDEVICE_PART1))
 	$(info $(SDDEVICE_PART2))
-	sgdisk --clear -g --new=1:2048:40M --new=2:512M:0 --typecode=1:3000 --typecode=2:8300 $(SD_DEVICE)
+	sgdisk --clear -g --new=1:2048:4M --new=2:512M:0 --typecode=1:3000 --typecode=2:8300 $(SD_DEVICE)
 	dd if=$(PAYLOAD) of=$(SDDEVICE_PART1) status=progress oflag=sync bs=1M
+	dd if=$(KERNEL) of=$(SDDEVICE_PART2) status=progress oflag=sync bs=1M
+
+CALL_LOGFILE ?= $(shell mktemp)
+call:
+	$(call log,info,Calling command on the CVA6 board)
+	ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \
+                -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+                 root@$(KEYSTONE_IP) $(KEYSTONE_COMMAND) 2>&1 | \
+                 grep -v "Warning: Permanently added" | tee -a $(CALL_LOGFILE)
+
+debug-connect:
+	$(call log,info,Connecting to OpenOCD)
+	$(BUILDROOT_BUILDDIR)/host/bin/riscv64-buildroot-linux-gnu-gdb \
+                -iex "set KEYSTONE=$(KEYSTONE)" \
+                -x $(KEYSTONE)/scripts/gdb/cva6.cfg
diff --git a/mkutils/plat/mpfs/run.mk b/mkutils/plat/mpfs/run.mk
@@ -20,7 +20,7 @@ run:
 
 CALL_LOGFILE ?= $(shell mktemp)
 call:
-	$(call log,info,Calling command in QEMU)
+	$(call log,info,Calling command on the MPFS board)
 	ssh -i $(BUILDROOT_BUILDDIR)/target/root/.ssh/id-rsa \
 		-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
 		 root@$(KEYSTONE_IP) $(KEYSTONE_COMMAND) 2>&1 | \