time-lazy-nozzle.sh

#!/bin/bash

set -e
set -x
set -o pipefail

date

exename="nozzle"
timestamp=$(date "+%Y.%m.%d-%H.%M.%S")
TIMING_HOME=$(pwd)
TIMING_HOST=$(hostname)
TIMING_DATE=$(date "+%Y-%m-%d %H:%M")
TIME_SINCE_EPOCH=$(date +%s)
TIMING_PLATFORM=$(uname)
TIMING_ARCH=$(uname -m)
TIMING_REPO="illinois-ceesd/timing.git"
TIMING_BRANCH="main"
TIMING_ENV_NAME="timing.main"
MIRGE_BRANCH="main"
DRIVER_REPO="illinois-ceesd/drivers_y1-nozzle"
DRIVER_BRANCH="main"
DRIVER_NAME="y1-production-nozzle-lazy"
SUMMARY_FILE_ROOT="${exename}_lazy"
YAML_FILE_NAME="${exename}-lazy-timings.yaml"
BATCH_OUTPUT_FILE="${SUMMARY_FILE_ROOT}_${timestamp}.out"
LOGDIR="${exename}_lazy_logs"
EXEOPTS="--lazy --log"
SQL_PATH="./log_data"

# -- Install conda env, dependencies and MIRGE-Com via *emirge*
# --- remove old run if it exists
if [ -f "INSTALL_MIRGECOM" ]
then
    if [ -d "emirge" ]
    then
        echo "Removing old timing run."
        mv -f emirge emirge.old
        rm -rf emirge.old &
    fi

    # --- grab emirge and install MIRGE-Com 
    git clone -b install-with-ssh https://github.com/illinois-ceesd/emirge.git
    cd emirge
    ./install.sh --branch=${MIRGE_BRANCH} --env-name=${TIMING_ENV_NAME}
    cd ../
fi
if [ -d "emirge/grudge" ]
then
    rm -f INSTALL_MIRGECOM
else
    echo "MIRGE-Com installation failed."
    exit 1
fi

# -- Activate the env we just created above
export EMIRGE_HOME="${TIMING_HOME}/emirge"
source ${EMIRGE_HOME}/config/activate_env.sh

cd emirge/mirgecom

# -- Grab and merge the branch with the case-dependent features
MIRGE_HASH=$(git rev-parse origin/${MIRGE_BRANCH})
Y1_HASH="$MIRGE_HASH"
cd ../..

# git merge origin/y1-production --no-edit

#  ------------- DISABLE ------------
# --- Grab the case driver repo
# rm -Rf ${DRIVER_NAME}
# git clone -b ${DRIVER_BRANCH} https://github.com/${DRIVER_REPO} ${DRIVER_NAME}
# cd ${DRIVER_NAME}/timing_run
# DRIVER_HASH=$(git rev-parse HEAD)

# cat <<EOF > timing_params.yaml
# nviz: 100
# nrestart: 100
# current_dt: 5e-8
# t_final: 1.e-6
# order:  1
# alpha_sc: 0.5
# s0_sc: -5.0
# kappa_sc: 0.5
# logDependent: 0
# EOF

# # --- Get an MD5Sum for the untracked timing driver
# DRIVER_MD5SUM="None"
# if command -v md5sum &> /dev/null
# then
#     DRIVER_MD5SUM=$(md5sum ./${exename}.py | cut -d " " -f 1)
# else
#     echo "Warning: No md5sum command found. Skipping  md5sum for untracked driver."
# fi

# # -- Run the case (platform-dependent)
# printf "Running on Host: ${TIMING_HOST}\n"
# date
# GPU_ARCH="Unknown"
# case $TIMING_HOST in
# 
#     # --- Run the timing test in a batch job on Lassen@LLC
#     lassen*)
#         echo "Resolved Host: Lassen"
#         TIMING_HOST="Lassen"
#        GPU_ARCH="GV100GL"
#         BATCH_SCRIPT_NAME="${exename}_timing_job.sh"
#         rm -f ${BATCH_SCRIPT_NAME}
#         rm -f timing-run-done
#         # ---- Generate a batch script for running the timing job
#         cat <<EOF > ${BATCH_SCRIPT_NAME}
# #!/bin/bash
# 
# #BSUB -nnodes 1
# #BSUB -G uiuc
# #BSUB -W 60
# #BSUB -q pdebug
# #BSUB -o ${BATCH_OUTPUT_FILE}
# 
# printf "Running with EMIRGE_HOME=${EMIRGE_HOME}\n"
# 
# source "${EMIRGE_HOME}/config/activate_env.sh"
# export PYOPENCL_CTX="port:tesla"
# export XDG_CACHE_HOME="/tmp/$USER/xdg-scratch"
# rm -rf \$XDG_CACHE_HOME
# rm -f timing-run-done
# which python
# conda env list
# jsrun -g 1 -a 1 -n 1 python -O -u -m mpi4py ./${exename}.py -i timing_params.yaml ${EXEOPTS}
# touch timing-run-done
# 
# EOF
#         chmod +x ${BATCH_SCRIPT_NAME}
#         # ---- Submit the batch script and wait for the job to finish
#         bsub ${BATCH_SCRIPT_NAME}
#         # ---- Wait 10 minutes right off the bat (the job is at least 10 min)
#         sleep 300
#         iwait=0
#         while [ ! -f ./timing-run-done ]; do
#             iwait=$((iwait+1))
#             if [ "$iwait" -gt 360 ]; then # give up after 1 hour
#                 printf "Timed out waiting on batch job.\n"
#                 exit 1 # skip the rest of the script
#             fi
#             sleep 10
#         done
#         ;;
# 
#     # --- Run the timing test on an unknown/generic machine
#     *)
#         printf "Host: Unknown\n"
#         PYOPENCL_TEST=port:pthread python -O -m mpi4py ./${exename}.py -i timing_params.yaml ${EXEOPTS}
#         ;;
# esac
# 
# date
# # -- Process the results of the timing run
# RUN_LOG_FILE="${SQL_PATH}/${exename}-rank0.sqlite"
# if [[ -f "${RUN_LOG_FILE}" ]]; then
# 
#     rm -f ${YAML_FILE_NAME}
#     SUMMARY_FILE_NAME="${SQL_PATH}/${SUMMARY_FILE_ROOT}_${timestamp}.sqlite"
#     rm -f ${SUMMARY_FILE_NAME}
# 
#     # --- Pull the timings out of the sqlite files generated by logging
#     runalyzer-gather ${SUMMARY_FILE_NAME} ${RUN_LOG_FILE}
#     CL_DEVICE=$(sqlite3 ${SUMMARY_FILE_NAME} 'SELECT cl_device_name FROM runs')
#     STARTUP_TIME=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(q("select $t_init.max").fetchall()[0][0])' | grep -v INFO)
#     FIRST_STEP=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[0:1]))' | grep -v INFO)
#     FIRST_10_STEPS=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[0:10]))' | grep -v INFO)
#     SECOND_10_STEPS=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(sum(p[0] for p in q("select $t_step.max").fetchall()[10:19]))' | grep -v INFO)
#     MAX_PYTHON_MEM_USAGE=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(max(p[0] for p in q("select $memory_usage_python.max").fetchall()))' | grep -v INFO)
#     MAX_GPU_MEM_USAGE=$(runalyzer -m ${SUMMARY_FILE_NAME} -c 'print(max(p[0] for p in q("select $memory_usage_gpu.max").fetchall()))' | grep -v INFO)
# 
#     # --- Create a YAML-compatible text snippet with the timing info
#     printf "run_date: ${TIMING_DATE}\nrun_host: ${TIMING_HOST}\n" > ${YAML_FILE_NAME}
#     printf "cl_device: ${CL_DEVICE}\n" >> ${YAML_FILE_NAME}
#     printf "run_epoch: ${TIME_SINCE_EPOCH}\nrun_platform: ${TIMING_PLATFORM}\n" >> ${YAML_FILE_NAME}
#     printf "run_arch: ${TIMING_ARCH}\ngpu_arch: ${GPU_ARCH}\n" >> ${YAML_FILE_NAME}
#     printf "mirge_version: ${MIRGE_HASH}\ny1_version: ${Y1_HASH}\n" >> ${YAML_FILE_NAME}
#    printf "driver_version: ${DRIVER_HASH}\ndriver_md5sum: ${DRIVER_MD5SUM}\n" >> ${YAML_FILE_NAME}
#     printf "time_startup: ${STARTUP_TIME}\ntime_first_step: ${FIRST_STEP}\n" >> ${YAML_FILE_NAME}
#    printf "time_first_10: ${FIRST_10_STEPS}\ntime_second_10: ${SECOND_10_STEPS}\n" >> ${YAML_FILE_NAME}
#     printf "max_python_mem_usage: ${MAX_PYTHON_MEM_USAGE}\n" >> ${YAML_FILE_NAME}
#     printf "max_gpu_mem_usage: ${MAX_GPU_MEM_USAGE}\n---\n" >> ${YAML_FILE_NAME}
#
#     # Users should set special keys for using git over
#     # ssh for security concerns. This snippet will use
#     # a pre-arranged ssh key if the user provides one
#     # and indicates it with the TESTING_SSH_KEY environment
#     # variable.
#     # ===== To create a key:
#     # - Run ssh-keygen:
#     # $ ssh-keygen
#     # [enter a <keyname> when prompted]
#     # - Put the key(s) in a /secure/filesystem/location:
#     # $ mv <keyname>* /secure/filesystem/location
#     # - Add the key to GIT:
#     # $ [browse to] https://github.com/illinois-ceesd/timing/settings/keys/new
#     # $ Choose (New SSH key)
#     # $ Paste in the contents of /secure/filesystem/location/<keyname>.pub
#     # - Set the ENV variable before using this script:
#     # $ export TESTING_SSH_KEY=/secure/filesystem/location/<keyname>
#     if [ ! -z "${TESTING_SSH_KEY}" ]; then
#         eval $(ssh-agent)
#         trap "kill $SSH_AGENT_PID" EXIT
#         ssh-add ${TESTING_SSH_KEY}
#     fi
# 
#     # --- Update the timing data in the repo
#     # ---- First, clone the timing repo
#     git clone -b ${TIMING_BRANCH} git@github.com:${TIMING_REPO}
#     # ---- Create the timing file if it does not exist
#     if [[ ! -f timing/${YAML_FILE_NAME} ]]; then
#         touch timing/${YAML_FILE_NAME}
#         (cd timing && git add ${YAML_FILE_NAME})
#     fi
#     # ---- Update the timing file with the current test data
#     cat ${YAML_FILE_NAME} >> timing/${YAML_FILE_NAME}
#     mkdir -p timing/${LOGDIR}
#     cp ${SUMMARY_FILE_NAME} timing/${LOGDIR}
#     cp ${BATCH_OUTPUT_FILE} timing/${LOGDIR}
#     cd timing
#     git add ${LOGDIR}/
#     # ---- Commit the new data to the repo
#     (git commit -am "Automatic commit: ${TIMING_HOST} ${TIMING_DATE}" && git push)
#     cd ../
# else
#    printf "Timing run did not produce the expected sqlite file: ${RUN_LOG_FILE}\n"
#     exit 1
# fi

date