Refactor ebpf test runner (#208)

* init commit

* move around files

* go code done

* modify makefile, refactor reader

* final cleanup

* cleanup, format

* try newer go version?

* try different compile args?

* try to deal with weird buffer errors

* tinker with timeouts

* fix script

* tinker with scripts

* try to get timing info

* tinker with profiling

* fix resource leak

* bad merge

* re-add DNS test

* clean up test bins, improve dns tests

* format

* tinker with overlay tests

* add back removed test, small cleanup

* try to move around files

* rename, part one

* rename, part two

* restore old file

* try again

* try rename again

* try other command

* try restore

* restore, 2

* completely restore old file

* rename old file

* modify in different commit

* update docs

* clean up network tests

.github/workflows/multikernel-tester.yml

@@ -64,7 +64,7 @@ jobs:
  - name: Install Go
  uses: actions/setup-go@v3
  with:
- go-version: '1.17'
+ go-version: '1.22'
  - name: Run tests
  run: make run-multikernel-test IMG_FILTER=${{ matrix.kernel_flavor }} ARCH=${{ inputs.architecture }} ARTIFACTS_PATH=${PWD}/artifacts
  - name: Prepare for archival

.gitignore

@@ -13,6 +13,7 @@ ctags
 cscope.out
 
 # Test results and other stuff in testing/
+testing/testrunner/testrunner.test
 testing/bpf-check-summary.txt
 testing/*.cpio
 testing/*.tar

Makefile

@@ -21,6 +21,9 @@ CONTAINER_LOCAL_TAG ?= ebpf-builder:${USER}-latest
 IMAGEPACK_REPOSITORY ?= ghcr.io/elastic/ebpf-imagepack
 IMAGEPACK_PULL_TAG ?= 20231006-0053
 
+TESTBIN_SRC = $(wildcard testing/test_bins/*.c)
+TESTBIN_PROGS = $(patsubst testing/test_bins/%.c,testing/test_bins/bin/${ARCH}/%,$(TESTBIN_SRC))
+
 ifdef BUILD_CONTAINER_IMAGE
  CONTAINER_IMAGE = ${CONTAINER_LOCAL_TAG}
 else
@@ -170,5 +173,14 @@ endif
  go install github.com/florianl/bluebox@b8590fb1850f56df6e6d7786931fcabdc1e9173d
  cd testing && ./run_tests.sh ${ARCH} ${ARTIFACTS_PATH} ${PWD}/kernel-images/${IMG_FILTER}/${ARCH}/*
 
+testbins: testbinpath $(TESTBIN_PROGS)
+
+testbinpath: 
+ mkdir -p testing/test_bins/bin/${ARCH}
+
+testing/test_bins/bin/${ARCH}/%: testing/test_bins/%.c
+ $(CC) -g -static -o $@ $<
+
 clean:
  ${SUDO} rm -rf artifacts-*
+ rm -r testing/test_bins/bin/*

non-GPL/Events/EventsTrace/EventsTrace.c

@@ -280,7 +280,10 @@ static void out_escaped_string(const char *value)
  break;
  default:
  if (!isascii(c) || iscntrl(c))
- printf("\\x%02x", c);
+ // \x is not a valid escape character in json,
+ // and something like '\xff' will break a remarkable number of JSON parsers.
+ // we have to print as '0xff'
+ printf("0x%02x", (uint8_t)c);
  else
  printf("%c", c);
  }

testing/README.md

@@ -55,6 +55,22 @@ By default `run_tests.sh` will pass `-j$(nproc)` to `parallel` (i.e. spin up as
 many jobs as there are CPU cores). You can change this by passing
 `-j <number of jobs>` to `run-tests.sh`.
 
+## Running tests locally
+
+This test framework leverages the go stdlib test suite, so all ebpf tests can
+be run as normal go tests, outside of the bluebox VMs:
+
+```
+#compile the test in advance, so we don't need root to have a go environment
+go test -c
+# run all tests
+sudo ./testrunner.test
+# run a single test
+sudo ./testrunner.test -test.run TestEbpf/Tcpv6ConnectionClose
+# run in verbose mode 
+sudo ./testrunner.test -test.run TestEbpf/Tcpv6ConnectionClose -test.v
+```
+
 ## Building Kernels
 
 A dockerized setup is provided at `kernel_builder/` to build mainline kernel

testing/run_tests.sh

@@ -8,7 +8,7 @@
 readonly PROGNAME=$(basename $0)
 readonly ARGS="$@"
 
-readonly SUCCESS_STRING="ALL BPF TESTS PASSED"
+readonly SUCCESS_STRING="exit status 0"
 readonly SUMMARY_FILE="bpf-check-summary.txt"
 readonly RESULTS_DIR="results"
 
@@ -70,8 +70,7 @@ EOF
 }
 
 main() {
- local arch=$1
- local artifacts="$2"
+
  local jobs=$(nproc)
 
  while getopts "j:" opt; do
@@ -85,6 +84,9 @@ main() {
  esac
  done
 
+ local arch=$1
+ local artifacts="$2"
+
  shift 2
 
  is_empty $arch \

testing/scripts/gen_initramfs.sh

@@ -37,7 +37,7 @@ build_testrunner() {
  pushd testrunner > /dev/null
 
  go clean
- GOARCH=$goarch go build
+ GOARCH=$goarch CGO_ENABLED=0 go test -c -o testrunner -ldflags '-extldflags "-static"'
 
  if [[ $? -ne 0 ]]
  then
@@ -78,7 +78,7 @@ invoke_bluebox() {
 
  local cmd="bluebox"
  cmd+=" -a $goarch"
- cmd+=" -e testrunner/testrunner"
+ cmd+=" -e testrunner/testrunner:-test.v"
  cmd+=" -r $eventstrace"
  cmd+=" -r $tcfiltertests"
  cmd+=" -r $tcfilterbpf"

testing/test_bins/common.h

@@ -45,3 +45,20 @@ void gen_pid_info_json(char *buf, size_t size)
  (pid_t)syscall(SYS_gettid), getppid(), getpid(), getsid(0), getpgid(0), cap_permitted,
  cap_effective);
 }
+
+int dump_info(int client_port, int server_port)
+{
+ char pid_info[8192];
+ gen_pid_info_json(pid_info, sizeof(pid_info));
+
+ char netns[128];
+ ssize_t nbytes;
+ CHECK(nbytes = readlink("/proc/self/ns/net", netns, sizeof(netns)), -1);
+ netns[nbytes] = '\0';
+
+ uint64_t netns_inode;
+ sscanf(netns, "net:[%lu]", &netns_inode);
+
+ printf("{ \"pid_info\": %s, \"client_port\": %d, \"server_port\": %d, \"netns\": %lu }\n",
+ pid_info, client_port, server_port, netns_inode);
+}

testing/test_bins/create_rename_delete_file_container.c

@@ -135,6 +135,11 @@ int main()
 
 cleanup:
  // Clean up directories created by child
+
+ // in the 5.10 test kernels, this umount call fails, so don't check.
+ // Can't reproduce the issue locally, but `unmount` operations on overlayfs have historically
+ // been quirky.
+ umount2(ovl_mountpoint, MNT_FORCE);
  CHECK(rm_recursive(ovl_mountpoint), -1);
  CHECK(rm_recursive(ovl_upperdir), -1);
  CHECK(rm_recursive(ovl_lowerdir), -1);

testing/test_bins/tcpv4_connect.c

@@ -25,23 +25,6 @@
 
 #define BOUND_PORT 2048
 
-int dump_info(int client_port, int server_port)
-{
- char pid_info[8192];
- gen_pid_info_json(pid_info, sizeof(pid_info));
-
- char netns[128];
- ssize_t nbytes;
- CHECK(nbytes = readlink("/proc/self/ns/net", netns, sizeof(netns)), -1);
- netns[nbytes] = '\0';
-
- uint64_t netns_inode;
- sscanf(netns, "net:[%lu]", &netns_inode);
-
- printf("{ \"pid_info\": %s, \"client_port\": %d, \"server_port\": %d, \"netns\": %lu }\n",
- pid_info, client_port, server_port, netns_inode);
-}
-
 int main()
 {
  struct sockaddr_in serveraddr;

testing/test_bins/tcpv6_connect.c

@@ -25,23 +25,6 @@
 
 #define BOUND_PORT 2048
 
-int dump_info(int client_port, int server_port)
-{
- char pid_info[8192];
- gen_pid_info_json(pid_info, sizeof(pid_info));
-
- char netns[128];
- ssize_t nbytes;
- CHECK(nbytes = readlink("/proc/self/ns/net", netns, sizeof(netns)), -1);
- netns[nbytes] = '\0';
-
- uint64_t netns_inode;
- sscanf(netns, "net:[%lu]", &netns_inode);
-
- printf("{ \"pid_info\": %s, \"client_port\": %d, \"server_port\": %d, \"netns\": %lu }\n",
- pid_info, client_port, server_port, netns_inode);
-}
-
 int main()
 {
  struct sockaddr_in6 serveraddr;

testing/test_bins/udp_send.c

@@ -13,6 +13,8 @@
 #include <string.h>
 #include <unistd.h>
 
+#include "common.h"
+
 int main(int argc, char *argv[])
 {
  struct sockaddr_in sin;
@@ -38,5 +40,7 @@ int main(int argc, char *argv[])
  else if (n != sizeof(buf))
  errx(1, "sendto: shortcount");
 
+ dump_info(53, 0);
+
  return (0);
 }