Taskfile.yml

######################################### DRIVER TASKS ############################################

_run-elan:
  desc: "Run through processing pipeline for Elan transcriptions"
  cmds:
    # Default extract stage, assuming data are cleaned and filtered.
    # Extracts data from all tiers in input files, which will flow all the way through the pipeline to the lexicon
    - task clean-output-folder tmp-makedir make-kaldi-subfolders
    - task elan-to-json
    - task clean-json > {{ .OUTPUT_PATH }}/tmp/{{ .CLEANED_FILTERED_DATA }}
    - task _build

_run-elan-split:
  desc: "Segment audio and text by Elan annotation start/end times then run through processing pipeline with default settings"
  cmds:
    # Extracts only from 'Phrase' tier, and skips annotations that have a corresponding annotation on a ref tier
    - task clean-output-folder tmp-makedir make-kaldi-subfolders
    - task split-eafs
    - task clean-json > {{ .OUTPUT_PATH }}/tmp/{{ .CLEANED_FILTERED_DATA }}
    - task _build

_run-trs:
  desc: "Run through processing pipeline for TRS transcriptions"
  cmds:
    - task clean-output-folder tmp-makedir make-kaldi-subfolders
    - task trs-to-json > {{ .OUTPUT_PATH }}/tmp/dirty.json
    - task clean-json > {{ .OUTPUT_PATH }}/tmp/{{ .CLEANED_FILTERED_DATA }}
    - task _build

_run-textgrid:
  desc: "Run through processing pipeline for TRS transcriptions"
  cmds:
    - task clean-output-folder tmp-makedir make-kaldi-subfolders
    - task textgrid-to-json > {{ .OUTPUT_PATH }}/tmp/dirty.json
    - task clean-json > {{ .OUTPUT_PATH }}/tmp/{{ .CLEANED_FILTERED_DATA }}
    - task _build


_build:
  desc: "Run through only the build stage with default settings (i.e. not the clean/filter/extract stage)"
  cmds:
    - task generate-kaldi-files
           generate-kaldi-configs
           copy-generated-files copy-phones-configs copy-helper-scripts
           gather-wavs extract-wavs

    - echo "######################## Build task completed without errors"

_train-test:
  desc: "Run Kaldi train and test stages on default settings"
  # dir: input/output/kaldi
  cmds:
    - cd {{ .OUTPUT_PATH }}/kaldi; ./run.sh


_show-lattice:
  desc: "Create pdf of the lattice graph"
  dir: input/output/kaldi
  cmds:
      - |
        for uttid in `cut -f1 -d " "  ../tmp/json_splitted/testing/text`; do
          ./utils/show_lattice.sh $uttid  ./exp/tri1/decode/lat.1.gz  ./data/lang/words.txt
        done

_infer:
  desc: "Run Kaldi inference on test data"
  cmds:
    - cd {{ .OUTPUT_PATH }}/kaldi; ../../../scripts/gmm-decode.sh

######################################### HELPER TASKS ############################################

generate-kaldi-files:
  desc: "Generate corpus-related files for Kaldi from JSON data"
  cmds:
    - task json-to-kaldi < {{ .OUTPUT_PATH }}/tmp/{{ .CLEANED_FILTERED_DATA }}

    - task make-wordlist < {{ .OUTPUT_PATH }}/tmp/{{ .CLEANED_FILTERED_DATA }} |
      task make-prn-dict > {{ .OUTPUT_PATH }}/tmp/lexicon.txt

    - task make-nonsil-phones > {{ .OUTPUT_PATH }}/tmp/nonsilence_phones.txt

generate-kaldi-configs:
  desc: "Generate config files for Kaldi from templates, populate with Taskvars"
  cmds:
    # Grab variables from Taskvars.yml and inject into mo command
    - KALDI_ROOT={{ .KALDI_ROOT }}
      HELPERS_PATH={{ .HELPERS_PATH }}
      CORPUS_PATH={{ .CORPUS_PATH }}
        mo < templates/path.sh > {{ .OUTPUT_PATH }}/tmp/path.sh

    - MFCC_SAMPLE_FREQUENCY={{ .MFCC_SAMPLE_FREQUENCY }}
      MFCC_FRAME_LENGTH={{ .MFCC_FRAME_LENGTH }}
      MFCC_LOW_FREQ={{ .MFCC_LOW_FREQ }}
      MFCC_HIGH_FREQ={{ .MFCC_HIGH_FREQ }}
      MFCC_NUM_CEPS={{ .MFCC_NUM_CEPS }}
        mo < templates/mfcc.conf > {{ .OUTPUT_PATH }}/tmp/mfcc.conf

    - DECODE_BEAM={{ .DECODE_BEAM }}
      DECODE_FIRST_BEAM={{ .DECODE_FIRST_BEAM }}
        mo < templates/decode.config > {{ .OUTPUT_PATH }}/tmp/decode.config

##################### Helpers for copying things

copy-generated-files:
  desc: "Copy generated files to appropriate (sub)directories under /output/kaldi"
  cmds:
    - cp {{ .OUTPUT_PATH }}/tmp/json_splitted/training/corpus.txt {{ .OUTPUT_PATH }}/kaldi/data/local/
    - cp {{ .OUTPUT_PATH }}/tmp/lexicon.txt {{ .OUTPUT_PATH }}/kaldi/data/local/dict/
    - cp {{ .OUTPUT_PATH }}/tmp/nonsilence_phones.txt {{ .OUTPUT_PATH }}/kaldi/data/local/dict/

    - cp {{ .OUTPUT_PATH }}/tmp/path.sh {{ .OUTPUT_PATH }}/kaldi/
    - cp {{ .OUTPUT_PATH }}/tmp/mfcc.conf {{ .OUTPUT_PATH }}/kaldi/conf/
    - cp {{ .OUTPUT_PATH }}/tmp/decode.config {{ .OUTPUT_PATH }}/kaldi/conf/

    # Note the default settings make the 'train' and 'test' folders identical (not anymore!)
    - cp {{ .OUTPUT_PATH }}/tmp/json_splitted/testing/segments {{ .OUTPUT_PATH }}/tmp/json_splitted/testing/text {{ .OUTPUT_PATH }}/tmp/json_splitted/testing/utt2spk {{ .OUTPUT_PATH }}/tmp/json_splitted/testing/wav.scp {{ .OUTPUT_PATH }}/kaldi/data/test/
    - cp {{ .OUTPUT_PATH }}/tmp/json_splitted/training/segments {{ .OUTPUT_PATH }}/tmp/json_splitted/training/text {{ .OUTPUT_PATH }}/tmp/json_splitted/training/utt2spk {{ .OUTPUT_PATH }}/tmp/json_splitted/training/wav.scp {{ .OUTPUT_PATH }}/kaldi/data/train/

copy-helper-scripts:
  desc: "Copy the necessary scripts from Kaldi"
  cmds:
    - cp {{ .HELPERS_PATH }}/templates/cmd.sh {{ .OUTPUT_PATH }}/kaldi/
    - cp {{ .HELPERS_PATH }}/templates/run.sh {{ .OUTPUT_PATH }}/kaldi/
    - cp {{ .HELPERS_PATH }}/templates/score.sh {{ .OUTPUT_PATH }}/kaldi/local/
    - cp -L -r {{ .KALDI_ROOT }}/egs/wsj/s5/steps {{ .OUTPUT_PATH }}/kaldi/steps
    - cp -L -r {{ .KALDI_ROOT }}/egs/wsj/s5/utils {{ .OUTPUT_PATH }}/kaldi/utils

copy-phones-configs:
  desc: "Copy provided silence/optional silence configuration files"
  cmds:
    - cp input/config/optional_silence.txt {{ .OUTPUT_PATH }}/kaldi/data/local/dict/
    - cp input/config/silence_phones.txt {{ .OUTPUT_PATH }}/kaldi/data/local/dict/

gather-wavs:
  desc: "Gather all wav files inside input/data into output/media.zip"
  cmds:
    # Tar up .wav files in order to keep folder structure the same
    # because Kaldi's wav.scp data file uses dir structure
    - cd {{ .CORPUS_PATH }}; tar cf {{ .HELPERS_PATH }}/{{ .OUTPUT_PATH }}/media.tar `find . | grep '\.wav'`

extract-wavs:
  desc: "Extract all wav files into kaldi folder"
  cmds:
    - tar xf {{ .OUTPUT_PATH }}/media.tar -C {{ .OUTPUT_PATH }}/kaldi
    - rm {{ .OUTPUT_PATH }}/media.tar

##################### Helpers for folders stuff

clean-output-folder:
  desc: "Delete all files and folders inside output directory"
  cmds:
    - rm -rf {{ .OUTPUT_PATH }}/*

tmp-makedir:
  desc: "Make the tmp directory, if it does not exist"
  cmds:
    - if [ ! -d {{ .OUTPUT_PATH }}/tmp ]; then mkdir {{ .OUTPUT_PATH }}/tmp; fi

tmp-delete:
  deps: [tmp-makedir]
  desc: "Delete all files in tmp directory"
  cmds:
    # Make directory non-empty, just in case it's just been initialised
    - cp /dev/null {{ .OUTPUT_PATH }}/null
    - rm -r {{ .OUTPUT_PATH }}/*

make-kaldi-subfolders:
  desc: "Makes subfolder structure which Kaldi expects"
  cmds:
    - mkdir -p {{ .OUTPUT_PATH }}/kaldi/data/local/dict
    - mkdir -p {{ .OUTPUT_PATH }}/kaldi/data/test
    - mkdir -p {{ .OUTPUT_PATH }}/kaldi/data/train
    - mkdir -p {{ .OUTPUT_PATH }}/kaldi/conf
    - mkdir -p {{ .OUTPUT_PATH }}/kaldi/local

##################### Helpers for generating things (mostly wrappers for Python scripts)

cat-all-json:
  desc: "Concatenate all .json files into one .json file"
  cmds:
    - jq -s '. | add'

make-nonsil-phones:
  desc: "Generate non-silence phones file from LETTER_TO_SOUND_PATH file defined in Taskfile.yml"
  cmds:
    - grep -v '^#' < {{ .LETTER_TO_SOUND_PATH }}
      | cut -d' ' -f2
      | grep -v '^$'
      | sort -u

elan-to-json:
  desc: "Convert a folder of .eaf files to a single JSON file"
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    - python3.6 {{ .SCRIPTS_PATH }}/elan_to_json.py --input_dir {{ .CORPUS_PATH }} --output_dir {{ .OUTPUT_PATH }} --tier {{ .TARGET_LANGUAGE_TIER }} --output_json {{ .OUTPUT_PATH }}/tmp/dirty.json

trs-to-json:
  desc: "Convert a folder of .trs files to a single JSON file"
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    - python3.6 {{ .SCRIPTS_PATH }}/trs_to_json.py --indir {{ .CORPUS_PATH }}

textgrid-to-json:
  desc: "Convert a folder of .textgrid files to a single JSON file"
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    # praatio is another that won't install because of the ssl error. use 3.4 for now
    - python3.4 {{ .SCRIPTS_PATH }}/textgrid_to_json.py --input_dir {{ .CORPUS_PATH }}

json-to-kaldi:
  desc: "Generate files for the Kaldi format"
  cmds:
    - python3.6 {{ .SCRIPTS_PATH }}/json_to_kaldi.py --output-folder="{{ .OUTPUT_PATH }}/tmp/json_splitted"

clean-json:
  desc: "Clean corpus of problematic characters before passing data to Kaldi"
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    - python3.6 {{ .SCRIPTS_PATH }}/clean_json.py  --infile {{ .OUTPUT_PATH }}/tmp/dirty.json

make-wordlist:
  desc: "Make a list of unique words that occur in the corpus"
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    - python3.6 {{ .SCRIPTS_PATH }}/make_wordlist.py

make-prn-dict:
  desc: "Make pronunciation dictionary"
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    - python3.6 {{ .SCRIPTS_PATH }}/make_prn_dict.py --config {{ .LETTER_TO_SOUND_PATH }}


resample-audio:
  desc: "Change audio to 16 bit 44.1kHz mono WAV"
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    - python3.6 {{ .SCRIPTS_PATH }}/resample_audio.py --corpus {{ .CORPUS_PATH }}

silence-audio:
  # Using numpy, so stick to Python3 (3.4.2 in the Docker image), but it means we can't be recursive
  desc: "Silence audio that has an annotion in matching Elan file 'Silence' tier"
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    - python3 {{ .SCRIPTS_PATH }}/silence_audio.py --corpus {{ .CORPUS_PATH }}

split-eafs:
  desc: "Read Elan files, slices matching WAVs by start and end times of annotations on a particular tier, outputting separate clips and text. Skips annotations with value '*PUB' on the main tier, or annotations that have a ref annotation on the 'Silence' tier."
  env:
    PYTHONIOENCODING: "utf-8"
  cmds:
    - python3.6 {{ .SCRIPTS_PATH }}/split_eafs.py --input_dir {{ .DIRTY_DATA_PATH }} --tier {{ .TARGET_LANGUAGE_TIER }} --silence_marker {{ .SILENCE_MARKER }} --silence_tier {{ .SILENCE_REF_TIER }} --output_json {{ .OUTPUT_PATH }}/tmp/dirty.json --output_audio_dir {{ .CORPUS_PATH }} --output_text_dir {{ .OUTPUT_PATH }}/tmp/labels