batch_convert_sxs_to_lvc.bash

#!/bin/bash

# Distributed under the MIT License.
# See LICENSE.txt for details.

# This script takes as arguments one or more paths. Each path provided
# should contain the files necessary to convert an SXS simulation to
# LVC format (rhOverM_Asymptotic_GeometricUnits_CoM.h5, Horizons.h5, and
# metadata.json), and each path should end in "/Lev#" or "/Res#", 
# where # is an integer. For each path, the script will launch a job that 
# converts the data in the path to LVC format, writing the output file to 
# the same path.
#
# To use this script, first edit the following parameters

# Path to JSON file listing parameters of all simulations in the public
# e.g. generated by
# github.com/sxs-collaboration/catalog_tools/get_sxs_public_metadata.py
catalog_json=/home/geoffrey/BBH/Catalog/zenodo_metadata/sxs_catalog.json

# Path to JSON file listing available resolutions in the public catalog, 
# e.g. generated by
# github.com/sxs-collaboration/catalog_tools/get_sxs_public_metadata.py
res_json=/home/geoffrey/BBH/Catalog/zenodo_metadata/sxs_catalog_resolutions.json

# Path to conversion script
convert_path=/home/geoffrey/BBH/Catalog/catalog_tools/convert_sxs_to_lvc.py

# Path to directory containing romspline python module
romspline_path=/home/geoffrey/BBH/Catalog/CatalogAnalysis

# Batch command used when submitting jobs to convert waveforms
batch_cmd="sbatch"

# Python environment command
# Run this command to activate the python environment you want to use
# Set to "echo date" or some other do-nothing command if this isn't 
# necessary on your system.
python_env_cmd="/home/geoffrey/apps/anaconda2/bin/activate root"

# Path to store batch submission scripts and stdout, stderr for each job
# (will be created if it doesn't exist)
output_path="/home/geoffrey/BBH/Catalog/convert_2018_public"

#############################################################
# SHOULD NOT NEED TO MODIFY ANYTHING BELOW THIS LINE
# (except, possibly, flags in cmd_to_run)
#############################################################

# First, make sure we can find the conversion script
convert_cmd=$(readlink -f ${convert_path})
if [[ -f ${convert_cmd} ]]
then
    echo "Found conversion script ${convert_cmd}"
else
    echo "ERROR: could not find ${convert_cmd}"
    exit 1
fi

# Next, make sure the two json files given above exist
if [[ -f ${catalog_json} ]]
then
    echo "Found ${catalog_json}"
else
    echo "ERROR: could not find ${catalog_json}"
    exit 1
fi

if [[ -f ${res_json} ]]
then
    echo "Found ${res_json}"
else
    echo "ERROR: could not find ${res_json}"
    exit 1
fi

# Check that conversion command works
source ${python_env_cmd}
echo "Using $(which python)"
echo "Testing python ${convert_cmd} --help"
$(python ${convert_cmd} --help &> /dev/null)
convert_test_status=$?
if [ ${convert_test_status} -ne 0 ]
then
    echo "ERROR: could not run conversion script"
fi

# Next, check that all paths have the expected data and name
for path in "$@"
do
    abs_path=$(readlink -f ${path})
    echo "Checking ${abs_path}"

    # Get resolution and paths to data to read
    base_path=$(basename ${path})
    resolution=$(echo ${base_path} | sed "s/Lev//" | sed "s/Res//")
    waveform_file=${abs_path}/rhOverM_Asymptotic_GeometricUnits_CoM.h5
    horizons_file=${abs_path}/Horizons.h5
    metadata_file=${abs_path}/metadata.json

    if [ "${resolution}" -eq "${resolution}" ]
    then
        echo "Resolution: ${resolution} (inferred from ${base_path})"
    else
        echo "ERROR: cannot infer resolution from path ${base_path}"
        exit 1
    fi

    # Check that required data files exist
    files_to_check=(${waveform_file} ${horizons_file} ${metadata_file})
    for file in ${files_to_check[@]}
    do
        if [[ -f "${file}" ]]
        then
            echo "Found $file"
        else
            echo "ERROR: Could not find ${file}"
            exit 1
        fi
    done
done

# Make the output directory if it doesn't exist
mkdir -p ${output_path}

# Finally, loop over all paths and submit a conversion job
typeset -i i CORESPERNODE
let CORESPERNODE=4 i=0

for path in "$@"
do
    abs_path=$(readlink -f ${path})
    echo "Processing ${abs_path}"
    base_path=$(basename ${path})
    resolution=$(echo ${base_path} | sed "s/Lev//" | sed "s/Res//")
    waveform_file=${abs_path}/rhOverM_Asymptotic_GeometricUnits_CoM.h5
    horizons_file=${abs_path}/Horizons.h5
    metadata_file=${abs_path}/metadata.json
    jobname="$(basename $(dirname ${abs_path})).${resolution}"

    # If $i=0, write a conversion script for this path
    if ((i==0)); then
        submit_file=${output_path}/${jobname}.sh
        echo "#!/bin/bash -" >> ${submit_file}
        echo "#SBATCH -o ${jobname}.stdout" >> ${submit_file}
        echo "#SBATCH -e ${jobname}.stderr" >> ${submit_file}
        echo "#SBATCH --ntasks 20" >> ${submit_file}
        echo "#SBATCH --cpus-per-task 1" >> ${submit_file}
        echo "#SBATCH -J ${jobname}" >> ${submit_file}
        echo "#SBATCH --nodes 1" >> ${submit_file}
        echo "#SBATCH -p orca-1" >> ${submit_file}
        echo "#SBATCH -t 120:00:00" >> ${submit_file}
        echo "#SBATCH -D ${output_path}" >> ${submit_file}
        echo "" >> ${submit_file}
    fi

    suffix="&"
    if ((i==CORESPERNODE-1)); then
        suffix=""
    fi

    cmd_to_run="$(which python) ${convert_cmd} --sxs_catalog_metadata ${catalog_json} --sxs_catalog_resolutions ${res_json} --modes all --resolution ${resolution} --sxs_data ${abs_path} --out_path ${abs_path}  --romspline_path ${romspline_path} ${suffix}"
    echo ${cmd_to_run} >> ${submit_file}

    if ((i==CORESPERNODE-1)); then
        echo "wait" >> ${submit_file}
        echo "" >> ${submit_file}
        echo "${batch_cmd} ${submit_file}"
        ${batch_cmd} ${submit_file}
        let i=0
    else
        let i++
    fi
done