hashed out for elcap

system-test/.gitignore

@@ -2,3 +2,4 @@ bats/
 
 dm/copy-in-copy-out/copy-in-copy-out.json
 env
+tmp*

system-test/Makefile

@@ -52,7 +52,7 @@ DM_PROFILE ?=
 export DM_PROFILE
 
 # Default bats command
-BATS = bats -j $(J) -T
+BATS = bats -j $(J) -T --print-output-on-failure  --verbose-run
 
 .PHONY: all
 all: init test dm

system-test/system-test.bats

@@ -30,23 +30,30 @@ fi
 
 # Flux command
 FLUX="flux run -l -N${N} --wait-event=clean"
+FLUX_A="flux alloc -N${N}"
 
 # Use a Flux queue
 if [ "${Q}" != "" ]; then
     FLUX+=" -q ${Q}"
+    FLUX_A+=" -q ${Q}"
 fi
 
 # Require specific rabbits
 if [ "${R}" != "" ]; then
     FLUX+=" --requires=hosts:${R}"
+    FLUX_A+=" --requires=hosts:${R}"
 fi
 
 # Command to run on the compute node via flux
 # TODO try set -x
 #COMPUTE_CMD="bash -c 'hostname'"
 COMPUTE_CMD='hostname'
 
-GB_PER_NODE=100
+# How much space to request for rabbit workflows per node
+GB_PER_NODE=300
+
+# For capacity tests, what is the threshold of available capacity vs the requested capacity
+CAPACITY_PERCENT=80
 
 function setup_file {
     # When using UUID in the DW name keyboard, just use the first 9 digits to make $DW_ env vars easier
@@ -56,6 +63,21 @@ function setup_file {
     export LUS_CAPACITY=$(($N * $GB_PER_NODE))
 }
 
+function setup {
+    # Create a tmpdir. To run flux jobs from this directory (which we do), I believe this needs to
+    # reside in the users homedir so the computes can reach the script. For now, use the working
+    # directory, which is assumed to be somewhere in a homedir.
+    # TODO: How is this going to work on dev systems with no NFS homedirs?
+    script_dir=$(pwd)
+    export TEST_TMPDIR=$(mktemp -d -p $script_dir)
+}
+
+function teardown {
+    if [[ -n "$TEST_TMPDIR" ]]; then
+        rm -rf $TEST_TMPDIR
+    fi
+}
+
 #----------------------------------------------------------
 # Simple Tests
 #----------------------------------------------------------
@@ -298,36 +320,66 @@ function setup_file {
 #----------------------------------------------------------
 # Capacity Tests
 #----------------------------------------------------------
-# bats test_tags=tag:xfs, tag:capacity
-@test "XFS - Capacity" {
-    ${FLUX_BATCH} --setattr=dw="\
-        #DW jobdw type=xfs name=xfs capacity=${GB_PER_NODE}GB" \
-        <<EOF
+function create_capacity_file {
+    wf_name=$1
+    runit_file="$TEST_TMPDIR/run-it.sh"
+    touch $runit_file
+    chmod +x $runit_file
+    cat << 'EOF' >> $runit_file
 #!/bin/bash
+requested_size=$1
+capacity_percent=$2
 
-df_output=$(flux run -N1 df $DW_JOB_xfs)
-# df_output=$(df -BG "$filesystem_path" 2>/dev/null | tail -1)
+echo "\$DW_JOB_<NAME>: $DW_JOB_<NAME>"
 
-# Check if df command succeeded
+df_output=$(flux run -N1 df -BG "$DW_JOB_<NAME>" 2>/dev/null | tail -1)
 if [ -z "$df_output" ]; then
     echo "Error: Invalid filesystem path or unable to retrieve information."
     exit 1
 fi
 
-# Extract available space
-# available_size=$(echo "$df_output" | awk '{print $4}' | sed 's/G//')
 available_size=$(echo "$df_output" | awk '{print $2}' | sed 's/G//')
+required_size=$(( requested_size * capacity_percent / 100 ))
 
-# Calculate 80% of the requested size
-required_size=$(( requested_size * 80 / 100 ))
+echo "requested_size: ${requested_size} GB"
+echo "required_size (${capacity_percent}%): ${required_size} GB"
+echo "available_size: ${available_size} GB"
 
-# Compare available size with required size
 if [ "$available_size" -ge "$required_size" ]; then
-    echo "Sufficient space available: ${available_size}G (>= 80% of ${requested_size}G)"
+    echo "Sufficient space available: ${available_size}G (>= ${capacity_percent}% of ${requested_size}G)"
     exit 0
 else
-    echo "Insufficient space: ${available_size}G (< 80% of ${requested_size}G)"
+    echo "Insufficient space: ${available_size}G (< ${capacity_percent}% of ${requested_size}G)"
     exit 1
 fi
 EOF
+
+    # replace the workflow name in the $DW_JOB_ env var
+    sed -i "s/<NAME>/$wf_name/g" $runit_file
+
+    echo $runit_file
+}
+
+# bats test_tags=tag:xfs, tag:capacity
+@test "XFS - Capacity" {
+    runit_file=$(create_capacity_file xfs)
+    ${FLUX_A} --setattr=dw="\
+        #DW jobdw type=xfs name=xfs capacity=${GB_PER_NODE}GB" \
+        $runit_file ${GB_PER_NODE} $CAPACITY_PERCENT
+}
+
+# bats test_tags=tag:gfs2, tag:capacity
+@test "GFS2 - Capacity" {
+    runit_file=$(create_capacity_file gfs2)
+    ${FLUX_A} --setattr=dw="\
+        #DW jobdw type=xfs name=gfs2 capacity=${GB_PER_NODE}GB" \
+        $runit_file ${GB_PER_NODE} $CAPACITY_PERCENT
+}
+
+# bats test_tags=tag:lustre, tag:capacity
+@test "Lustre - Capacity" {
+    runit_file=$(create_capacity_file lustre)
+    ${FLUX_A} --setattr=dw="\
+        #DW jobdw type=xfs name=lustre capacity=${LUS_CAPACITY}GB" \
+        $runit_file ${LUS_CAPACITY} $CAPACITY_PERCENT
 }