Merge pull request #97 from lanl/gshipman

update branson and parthenon weak scaling scripts
lanl · Apr 16, 2024 · 40f4e16 · 40f4e16
2 parents 2ae20eb + 230e4df
commit 40f4e16
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 6 deletions.
diff --git a/utils/branson/weak_scale_cpu_threads.sh b/utils/branson/weak_scale_cpu_threads.sh
@@ -29,6 +29,8 @@ workdir=${SUFFIX}_${SLURM_JOB_ID}
 mkdir ${workdir}
 cd ${workdir}
 cp ../../inputs/${input} .
+export OMP_PROC_BIND=spread
+export OMP_PLACES=threads
 
 echo ${input}
 sed -i 's/<t_stop>.*<\/t_stop>/<t_stop>'.010'\<\/t_stop>/g' ${input}
@@ -57,8 +59,8 @@ for nodes in 1 4 8 16 32 40 64 96; do
     outfile=$(printf "weakscale_${SUFFIX}-%d-%d.out" ${size} ${threads})
     errfile=$(printf "weakscale_${SUFFIX}-%d-%d.err" ${size} ${threads})
     grep photons ${input}
-    MPICH_SMP_SINGLE_COPY_MODE=NONE FI_CXI_RX_MATCH_MODE=software OMP_NUM_THREADS=14
-    srun ${ht}  -N ${nodes} -n ${ranks} --ntasks-per-node=${numas} -c ${cores} -o ${outfile} -e ${errfile}  ../BRANSON ./3D_hohlraum_multi_node.xml  
+    MPICH_SMP_SINGLE_COPY_MODE=NONE FI_CXI_RX_MATCH_MODE=software 
+    OMP_NUM_THREADS=${cores}  srun ${ht}  -N ${nodes} -n ${ranks} --ntasks-per-node=${numas} -c ${cores} -o ${outfile} -e ${errfile}  ../BRANSON ./3D_hohlraum_multi_node.xml  
     fom=$(grep 'Photons Per Second (FOM): ' ${outfile} | cut -d ':' -f 2 | xargs)
     echo ${fom}
 

diff --git a/utils/branson/weak_scale_xroads.batch b/utils/branson/weak_scale_xroads.batch
@@ -4,5 +4,7 @@
 #SBATCH --time=00:30:00
 #SBATCH -p hbm
 echo "$@"
-export CALI_CONFIG="spot,time.exclusive,profile.mpi"
+#export CALI_CONFIG="spot,time.exclusive,profile.mpi"
+export CALI_CONFIG="spot,time.exclusive"
+
 ./weak_scale_cpu_threads.sh "$@"
diff --git a/utils/parthenon/weak_scale_cpu_threads.sh b/utils/parthenon/weak_scale_cpu_threads.sh
@@ -27,9 +27,10 @@ workdir=${SUFFIX}_${SLURM_JOB_ID}
 mkdir ${workdir}
 cd ${workdir}
 cp ../../../../benchmarks/burgers/${input} . 
+sed -i 's/pack_size = .*/pack_size = '${cores}'/g' ${input}
 NXs=(32 64 128 256 432 512)  
 NXBs=(16 32)
-NLIM=20
+NLIM=100
 NLVL=3
 NODES=(1 4 8 16 32 64 96)
 export OMP_PROC_BIND=spread
@@ -55,7 +56,7 @@ for N in ${NODES[@]}; do
 	        fi
             echo "ranks: ${ranks} threads: ${threads}"
             echo "running ${NXB} blocksize, ${NX} grid, ${N} nodes, ${ranks} ranks"
-            OMP_NUM_THREADS=${cores} srun ${BINDOPTS}  --exclusive -n ${ranks} -N ${N}  --ntasks-per-node=${numas} -c ${cores} ../burgers-benchmark -i ./burgers.pin parthenon/mesh/nx1=${NX}  parthenon/mesh/nx2=${NX} parthenon/mesh/nx3=${NX} parthenon/meshblock/nx1=${NXB}  parthenon/meshblock/nx2=${NXB} parthenon/meshblock/nx3=${NXB} parthenon/time/nlim=${NLIM} parthenon/mesh/numlevel=${NLVL} |& tee scale-cpu.$NXB.$NX.$N.out
+            OMP_NUM_THREADS=${cores} srun ${BINDOPTS}  --exclusive  --distribution=block:block ${ht} -n ${ranks} -N ${N}  --ntasks-per-node=${numas} -c ${cores} ../burgers-benchmark -i ./burgers.pin parthenon/mesh/nx1=${NX}  parthenon/mesh/nx2=${NX} parthenon/mesh/nx3=${NX} parthenon/meshblock/nx1=${NXB}  parthenon/meshblock/nx2=${NXB} parthenon/meshblock/nx3=${NXB} parthenon/time/nlim=${NLIM} parthenon/mesh/numlevel=${NLVL} |& tee scale-cpu.$NXB.$NX.$N.out
 	    done
     done
 done
diff --git a/utils/parthenon/weak_scale_xroads.batch b/utils/parthenon/weak_scale_xroads.batch
@@ -4,5 +4,5 @@
 #SBATCH --time=00:30:00
 #SBATCH -p hbm
 echo "$@"
-#export CALI_CONFIG="spot,time.exclusive,profile.mpi"
+export CALI_CONFIG="spot,time.exclusive,profile.mpi"
 ./weak_scale_cpu_threads.sh "$@"