Add multi-node configuration and enable more tests in single-node mode.

experiment/common/setup.sh

@@ -72,8 +72,12 @@ pushd legion
 build_legion_config debug_single Debug
 build_legion_config spy_single Debug -DLegion_SPY=ON
 build_legion_config release_single Release
+build_legion_config debug_multi Debug "-DLegion_NETWORKS=gasnetex -DLegion_EMBED_GASNet=ON -DGASNet_CONDUIT=$FUZZER_CONDUIT -DLegion_EMBED_GASNet_CONFIGURE_ARGS=--disable-kind-cuda-uva"
+build_legion_config release_multi Release "-DLegion_NETWORKS=gasnetex -DLegion_EMBED_GASNet=ON -DGASNet_CONDUIT=$FUZZER_CONDUIT -DLegion_EMBED_GASNet_CONFIGURE_ARGS=--disable-kind-cuda-uva"
 popd
 
 build_fuzzer_config debug_single RelWithDebInfo
 build_fuzzer_config spy_single RelWithDebInfo
 build_fuzzer_config release_single Release
+build_fuzzer_config debug_multi RelWithDebInfo
+build_fuzzer_config release_multi Release

experiment/sapling/env.sh

@@ -1,4 +1,5 @@
 export FUZZER_MACHINE=sapling
 export FUZZER_THREADS=20
+export FUZZER_CONDUIT=ibv
 
 export CC=gcc CXX=g++

experiment/sapling/run_all_tests.sh

@@ -10,19 +10,32 @@ fi
 root_dir="$(dirname "$(dirname "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")")")"
 cd "$root_dir"
 
-# Currently we give everything equal testing
-export FUZZER_TEST_COUNT=10000
 export FUZZER_OP_COUNT=1000
 export FUZZER_EXTRA_FLAGS="-ll:util 2 -ll:cpu 3"
-export FUZZER_LAUNCHER="srun -n 1 --overlap"
 
 function run_fuzzer_config {
     config_name="$1"
+    mode="$2"
 
     fuzzer_exe="$PWD/build_${config_name}/src/fuzzer"
 
-    FUZZER_EXE="$fuzzer_exe" sbatch --nodes 1 "experiment/$FUZZER_MACHINE/sbatch_fuzzer.sh"
+    if [[ $mode = single ]]; then
+        test_count=100000
+        launcher="srun -n 1 --pty"
+    elif [[ $mode = multi ]]; then
+        # We can't do as many tests in multi-node mode because SLURM has a
+        # hard upper bound on the number of steps per job.
+        test_count=10000
+        launcher="srun -n 2 --overlap"
+    else
+        echo "Don't recognize fuzzer mode $mode"
+        exit 1
+    fi
+
+    FUZZER_EXE="$fuzzer_exe" FUZZER_MODE=$mode FUZZER_TEST_COUNT=$test_count FUZZER_LAUNCHER="$launcher" sbatch --nodes 1 "experiment/$FUZZER_MACHINE/sbatch_fuzzer.sh"
 }
 
-run_fuzzer_config debug_single
-run_fuzzer_config release_single
+run_fuzzer_config debug_single single
+run_fuzzer_config release_single single
+run_fuzzer_config debug_multi multi
+run_fuzzer_config release_multi multi

experiment/sapling/sbatch_fuzzer.sh

@@ -11,4 +11,25 @@ ulimit -S -c 0 # disable core dumps
 
 set -x
 
-./runner.py --fuzzer="$FUZZER_EXE" -j${FUZZER_THREADS:-4} -n${FUZZER_TEST_COUNT:-1000} -o${FUZZER_OP_COUNT:-1000} --extra="$FUZZER_EXTRA_FLAGS" --launcher="$FUZZER_LAUNCHER"
+launcher=
+
+fuzzer_flags=(
+    --fuzzer="$FUZZER_EXE"
+    -j${FUZZER_THREADS:-4}
+    -n${FUZZER_TEST_COUNT:-1000}
+    -o${FUZZER_OP_COUNT:-1000}
+    --extra="$FUZZER_EXTRA_FLAGS"
+)
+
+if [[ $FUZZER_MODE = single ]]; then
+    launcher="$FUZZER_LAUNCHER"
+elif [[ $FUZZER_MODE = multi ]]; then
+    fuzzer_flags+=(
+        --launcher="$FUZZER_LAUNCHER"
+    )
+else
+    echo "Don't recognize fuzzer mode $FUZZER_MODE"
+    exit 1
+fi
+
+$launcher ./runner.py "${fuzzer_flags[@]}"