01.merge-text-meta

#!/bin/bash
#SBATCH --job-name=merge-text-meta
#SBATCH --partition="small"
#SBATCH --time=72:00:00
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=128
#SBATCH --mem-per-cpu=1750
#SBATCH --output=logs/%x.out
source .env
set -euo pipefail

COLL=$1
INPUT_DIR=${COLLECTIONS[$COLL]}
export OUTPUT_DIR=$WORKSPACE/merged/$COLL
mkdir -p $OUTPUT_DIR

echo Num CPUS $SLURM_CPUS_ON_NODE
run-merge (){
    local coll=$1
    local dir=$2
    local filenum=$3
    python scripts/merge-meta-text.py $coll $dir $OUTPUT_DIR/$filenum
}
export -f run-merge

binddirs=`echo $INPUT_DIR* | tr ' ' ','`,$OUTPUT_DIR,$(pwd -P)
# Run every batch in parallel
# each batch in the output will be renamed to the job number
$PROFILER \
singularity exec --bind $binddirs --pwd $(pwd -P) monotextor.sif \
parallel -j$SLURM_CPUS_ON_NODE --will-cite --halt now,fail=1 \
    run-merge $COLL {} {\#} ::: `find $INPUT_DIR* -maxdepth 1 -mindepth 1 -type d`