fddev: add CI smoke test

.github/workflows/on_pull_request.yml

@@ -9,7 +9,3 @@ jobs:
   tests:
     uses: ./.github/workflows/tests.yml
     secrets: inherit
-
-  tests-fddev:
-    uses: ./.github/workflows/tests_fddev.yml
-    secrets: inherit

.github/workflows/tests.yml

@@ -38,6 +38,7 @@ jobs:
 
       - uses: ./.github/actions/deps
       - uses: ./.github/actions/hugepages
+      - uses: dtolnay/[email protected]
 
       - name: Run tests
         run: |

config/everything.mk

@@ -1,8 +1,8 @@
 MAKEFLAGS += --no-builtin-rules
 MAKEFLAGS += --no-builtin-variables
 .SUFFIXES:
-.PHONY: all info bin rust include lib unit-test fuzz-test help clean distclean asm ppp show-deps
-.PHONY: run-unit-test run-script-test run-fuzz-test run-integration-test
+.PHONY: all info bin rust include lib unit-test integration-test fuzz-test help clean distclean asm ppp show-deps
+.PHONY: run-unit-test run-integration-test run-script-test run-fuzz-test
 .PHONY: seccomp-policies cov-report dist-cov-report
 .SECONDARY:
 .SECONDEXPANSION:
@@ -13,7 +13,7 @@ CPPFLAGS+=-DFD_BUILD_INFO=\"$(OBJDIR)/info\"
 CPPFLAGS+=$(EXTRA_CPPFLAGS)
 
 # Auxiliary rules that should not set up dependencies
-AUX_RULES:=clean distclean help show-deps run-unit-test cov-report dist-cov-report
+AUX_RULES:=clean distclean help show-deps run-unit-test run-integration-test cov-report dist-cov-report
 
 all: info bin include lib unit-test fuzz-test
 
@@ -45,15 +45,17 @@ help:
 	# SCRUB           = $(SCRUB)
 	# FUZZFLAGS       = $(FUZZFLAGS)
 	# EXTRAS_CPPFLAGS = $(EXTRA_CPPFLAGS)
-	# Explicit goals are: all bin include lib unit-test help clean distclean asm ppp
+	# Explicit goals are: all bin include lib unit-test integration-test help clean distclean asm ppp
 	# "make all" is equivalent to "make bin include lib unit-test fuzz-test"
 	# "make info" makes build info $(OBJDIR)/info for the current platform (if not already made)
 	# "make bin" makes all binaries for the current platform (except those requiring the Rust toolchain)
 	# "make include" makes all include files for the current platform
 	# "make lib" makes all libraries for the current platform
 	# "make unit-test" makes all unit-tests for the current platform
+	# "make integration-test" makes all integration-tests for the current platform
 	# "make rust" makes all binaries for the current platform that require the Rust toolchain
 	# "make run-unit-test" runs all unit-tests for the current platform. NOTE: this will not (re)build the test executables
+	# "make run-integration-test" runs all integration-tests for the current platform. NOTE: this will not (re)build the test executables
 	# "make help" prints this message
 	# "make clean" removes editor temp files and the current platform build
 	# "make distclean" removes editor temp files and all platform builds
@@ -89,6 +91,12 @@ run-unit-test:
 	#######################################################################
 	contrib/test/run_unit_tests.sh --tests $(OBJDIR)/unit-test/automatic.txt $(TEST_OPTS)
 
+run-integration-test:
+	#######################################################################
+	# Running integration tests
+	#######################################################################
+	contrib/test/run_integration_tests.sh --tests $(OBJDIR)/integration-test/automatic.txt $(TEST_OPTS)
+
 ##############################
 # Usage: $(call make-lib,name)
 
@@ -175,7 +183,9 @@ add-test-scripts = $(foreach script,$(1),$(eval $(call _add-script,unit-test,$(s
 # Usage: $(call make-bin,name,objs,libs)
 # Usage: $(call make-shared,name,objs,libs)
 # Usage: $(call make-unit-test,name,objs,libs)
+# Usage: $(call make-integration-test,name,objs,libs)
 # Usage: $(call run-unit-test,name,args)
+# Usage: $(call run-integration-test,name,args)
 # Usage: $(call make-fuzz-test,name,objs,libs)
 
 # Note: The library arguments require customization of each target
@@ -212,6 +222,16 @@ $(OBJDIR)/unit-test/automatic.txt:
 	$(MKDIR) "$(OBJDIR)/unit-test"
 	@$(foreach test,$(RUN_UNIT_TEST),echo $(test)>>$@;)
 
+# Generate list of automatic integration tests from $(call run-integration-test,...)
+integration-test: $(OBJDIR)/integration-test/automatic.txt
+define _run-integration-test
+RUN_INTEGRATION_TEST+=$(OBJDIR)/integration-test/$(1)
+endef
+$(OBJDIR)/integration-test/automatic.txt:
+	$(MKDIR) "$(OBJDIR)/integration-test"
+	@$(foreach test,$(RUN_INTEGRATION_TEST),echo $(test)>>$@;)
+	$(TOUCH) "$@"
+
 ifndef FD_HAS_FUZZ
 FUZZ_EXTRA:=$(OBJDIR)/lib/libfd_fuzz_stub.a
 endif
@@ -247,6 +267,8 @@ make-bin-rust  = $(eval $(call _make-exe,$(1),$(2),$(3),rust,bin))
 make-shared    = $(eval $(call _make-exe,$(1),$(2),$(3),lib,lib,-shared))
 make-unit-test = $(eval $(call _make-exe,$(1),$(2),$(3),unit-test,unit-test))
 run-unit-test  = $(eval $(call _run-unit-test,$(1)))
+make-integration-test = $(eval $(call _make-exe,$(1),$(2),$(3),integration-test,integration-test))
+run-integration-test  = $(eval $(call _run-integration-test,$(1)))
 make-fuzz-test = $(eval $(call _fuzz-test,$(1),$(2),$(3)))
 
 ##############################
@@ -402,12 +424,6 @@ MACHINE=$(MACHINE) \
 LLVM_PROFILE_FILE="$(OBJDIR)/cov/raw/script_test-%p.profraw" \
 contrib/test/run_script_tests.sh
 
-run-integration-test: fddev
-	mkdir -p "$(OBJDIR)/cov/raw" && \
-OBJDIR=$(OBJDIR) \
-LLVM_PROFILE_FILE="$(OBJDIR)/cov/raw/integration_tests.profraw" \
-contrib/test/run_integration_tests.sh
-
 seccomp-policies:
 	$(FIND) . -name '*.seccomppolicy' -exec $(PYTHON) contrib/codegen/generate_filters.py {} \;
 

contrib/test/ci_tests.sh

@@ -29,9 +29,12 @@ for MACHINE in ${MACHINES[*]}; do
   OBJDIR="$(make help | grep OBJDIR | awk '{print $4}')"
   OBJDIRS+=( "${OBJDIR}" )
   make clean --silent >/dev/null
-  contrib/make-j
+  contrib/make-j all integration-test
   if [[ "$NOTEST" != 1 ]]; then
     make run-unit-test
+    if [[ "$EXTRAS" != *"ubsan"* && "$EXTRAS" != *"asan"* ]]; then
+      make run-integration-test
+    fi
     make run-fuzz-test
     make run-script-test
     if [[ "$HAS_LLVM_COV" == 1 ]]; then

contrib/test/run_integration_tests.sh

@@ -1,13 +1,212 @@
-#!/bin/bash
+#!/usr/bin/env bash
+# run_integration_tests.sh is like run_unit_tests.sh except that tests
+# are not run concurrently, only one integration test will currently run
+# at a time.
+#
+# WARNING: These tests might change your system configuration.
 
-# WARNING: These tests will destroy your system configuration.
+set -eou pipefail
 
-export OBJDIR
-export LLVM_PROFILE_FILE
+# Defaults
 
-set -xeuo pipefail
+TESTS=
+VERBOSE=0
 
-cd "$(dirname "$0")/../.."
+# Read command-line args
+
+while [[ $# -gt 0 ]]; do
+
+  FLAG="$1"
+  shift 1
+
+  case "$FLAG" in
+    "--tests")
+      TESTS="$1"
+      shift 1
+      ;;
+    "-v")
+      VERBOSE=1
+      ;;
+    *)
+      echo "Unknown flag: $FLAG" >&2
+      exit 1
+      ;;
+  esac
+
+done
+
+if [[ -z "$TESTS" && -z "${RECURSE_GUARD:-}" ]]; then
+  # No tests given, so indirect test execution through Make and retry.
+  # This ensures that we select the correct build dir for the current
+  # environment ($CC, $MACHINE, etc).
+  # Make then re-executes this file with the proper parameters.
+
+  export RECURSE_GUARD=1
+  exec make run-integration-test TEST_OPTS="$*"
+fi
+
+# Ensure we schedule at most one job at a time
+AVAILABLE_JOBS=1
+
+# Clean up process tree on exit
+trap 'exit' INT QUIT TERM
+
+# Track list of PIDs of child processes
+declare -A PIDS=()
+# Remember unit name of each PID
+declare -A PID2UNIT=()
+# Remember logfile name of each PID
+declare -A PID2LOG=()
+
+# Read in list of automatic integration tests to schedule as jobs
+declare -a TEST_LIST=()
+
+if [[ -s "$TESTS" ]]; then
+  while read -r line; do
+    if [[ "$line" =~ ^[[:space:]]*# ]]; then
+      continue
+    fi
+    TEST_LIST+=( "$line" )
+  done < <(grep -v '^#' "$TESTS")
+fi
+
+echo "test.sh: Scheduling ${#TEST_LIST[@]} tests in sequence" >&2
+
+rc_path () {
+  echo "/tmp/.pid-$1.rc"
+}
+
+runner () {
+  local pid="$BASHPID"
+  local prog="$1"
+  local log="$2"
+  local logfull="${log%.log}-full.log"
+  shift 2
+
+  # Create coverage dir, in case it's used
+  local covdir; covdir="$(dirname "$prog")/../cov/raw"
+  mkdir -p "$covdir"
+  # Set up coverage file (no-op for non-instrumented binary)
+  local LLVM_PROFILE_FILE
+  LLVM_PROFILE_FILE="$covdir/$(basename "$prog").profraw"
+
+  set +e
+  local elapsed
+  elapsed="$({                 \
+    time                       \
+    LLVM_PROFILE_FILE="$LLVM_PROFILE_FILE" \
+    "sudo"                     \
+      "$prog"                  \
+      "$@"                     \
+      --log-path "$logfull"    \
+      --log-level-stderr 3     \
+      >/dev/null               \
+      2>"$log"                 \
+    ; } \
+    2>&1 >/dev/null \
+    | grep real \
+    | awk '{print $2}'
+  )"
+  local ret="$?"
+
+  local rcpath; rcpath="$(rc_path "$pid")"
+  {
+    echo "$ret"
+    echo "$elapsed"
+  } > "$rcpath"
+}
+
+# dispatch numa_idx cpu_idx cmdline...
+#   Fork task
+dispatch () {
+  # Craft command line args
+  local prog="$1"
+  local progname="${prog##*/}"
+  shift 1
+
+  # Create log dir
+  local logdir
+  logdir="$(dirname "$(dirname "$prog")")/log/$progname"
+  mkdir -p "$logdir"
+  local log; log="$logdir/$(date -u +%Y%m%d-%H%M%S).log"
+
+  if [[ "$VERBOSE" == 1 ]]; then
+    echo "test.sh: NUMA $numa_idx: $progname" >&2
+  fi
+
+  # Dispatch
+  runner "$prog" "$log" "$@" &
+  local pid="$!"
+
+  # Remember PID
+  PIDS["$pid"]=$pid
+  PID2UNIT["$pid"]="$progname"
+  PID2LOG["$pid"]="$log"
+}
+
+# sow
+#   schedule tasks until max concurrency reached
+sow () {
+  if [[ "${#TEST_LIST[@]}" -eq 0 ]]; then return; fi
+  if [[ "${AVAILABLE_JOBS}" -eq 0 ]]; then return; fi
+
+  # Found a free CPU!
+  local test="${TEST_LIST[0]}"
+  TEST_LIST=( "${TEST_LIST[@]:1}" )
+  AVAILABLE_JOBS="$(( AVAILABLE_JOBS - 1 ))"
+  dispatch "$test"
+}
+
+FAIL_CNT=0
+
+# reap
+#   wait for a job to finish
+reap () {
+  wait -n "${PIDS[@]}"
+  # Clean up finished jobs
+  for pid in "${PIDS[@]}"; do
+    if [[ ! -d "/proc/$pid" ]]; then
+      # Job finished
+      local rcfile; rcfile="$(rc_path "$pid")"
+      local rc
+      local elapsed
+      {
+        IFS= read -r rc
+        IFS= read -r elapsed
+      } < "$rcfile"
+      local unit="${PID2UNIT["$pid"]}"
+      local log="${PID2LOG["$pid"]}"
+      local logfull="${log%.log}-full.log"
+      unset PIDS["$pid"]
+      unset PID2UNIT["$pid"]
+      unset PID2LOG["$pid"]
+      AVAILABLE_JOBS="$(( AVAILABLE_JOBS + 1 ))"
+      if [[ "$rc" -ne 0 ]]; then
+        FAIL_CNT="$(( FAIL_CNT + 1 ))"
+        printf "\033[0;31mFAIL\033[0m%12s   %s (exit %d): %s\n" "$elapsed" "$unit" "$rc" "$logfull" >&2
+        grep -sv "Log at" "$log" | sed -e "$(printf "s/^/%19s%-20s /" '' "$unit")" || true >&2
+      else
+        printf "\033[0;32mOK  \033[0m%12s   %s\n"               "$elapsed" "$unit" >&2
+      fi
+    fi
+  done
+}
+
+while [[ "${#TEST_LIST[@]}" -gt 0 ]]; do
+  sow
+  reap
+done
+while [[ "${#PIDS[@]}" -gt 0 ]]; do
+  reap
+done
+
+if [[ "$FAIL_CNT" -gt 0 ]]; then
+  echo -e "\033[0;31mFAIL\033[0m ($FAIL_CNT failure)" >&2
+  exit 1
+else
+  echo -e "\033[0;32mPASS\033[0m" >&2
+  exit 0
+fi
 
 # TODO add fddev integration tests here
 

src/app/fdctl/fdctl.h

@@ -37,6 +37,7 @@ typedef union {
   } configure;
 
   struct {
+    int  parent_pipefd;
     int  monitor;
     int  no_configure;
     int  no_solana_labs;