02.split-lang

#!/bin/bash
#SBATCH --job-name=split-lang
#SBATCH --partition="small"
#SBATCH --time=72:00:00
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=48
#SBATCH --mem-per-cpu=1750
#SBATCH --output=logs/%x.out
source .env
set -euo pipefail

COLL=$1
if [[ ! -n ${COLLECTIONS[$COLL]} ]]; then
    echo "'$COLL' does not exist"
    exit 1
fi
export INPUT_DIR=$WORKSPACE/merged/$COLL
export OUTPUT_DIR=$WORKSPACE/batches/$COLL
mkdir -p $OUTPUT_DIR

binddirs=$INPUT_DIR,$OUTPUT_DIR,$(pwd -P)
# split documents in lang folders
$PROFILER \
singularity exec --bind $binddirs --pwd $(pwd -P) monotextor.sif \
python scripts/split-lang.py $OUTPUT_DIR $INPUT_DIR/*.docs.zst

# clean the merged directory, it is just intermediate
rm -r $WORKSPACE/merged/$COLL