diff --git a/egs/commonvoice/ASR/local/compute_fbank_commonvoice_splits.py b/egs/commonvoice/ASR/local/compute_fbank_commonvoice_splits.py index d1af258cac..fd565da83e 100755 --- a/egs/commonvoice/ASR/local/compute_fbank_commonvoice_splits.py +++ b/egs/commonvoice/ASR/local/compute_fbank_commonvoice_splits.py @@ -45,7 +45,7 @@ def get_args(): parser.add_argument( "--subset", type=str, - default="train" + default="train", help="""Dataset parts to compute fbank. """, ) diff --git a/egs/commonvoice/ASR/prepare.sh b/egs/commonvoice/ASR/prepare.sh index d1eb0da542..905c02b7f1 100755 --- a/egs/commonvoice/ASR/prepare.sh +++ b/egs/commonvoice/ASR/prepare.sh @@ -10,6 +10,12 @@ stop_stage=100 # This is to avoid OOM during feature extraction. num_splits=1000 +# In case you want to use all validated data +use_validated=false + +# In case you are willing to take the risk and use invalidated data +use_invalidated=false + # We assume dl_dir (download dir) contains the following # directories and files. If not, they will be downloaded # by this script automatically. @@ -102,19 +108,23 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then if [ ! -e data/${lang}/manifests/.cv-${lang}.done ]; then lhotse prepare commonvoice --language $lang -j $nj $dl_dir/$release data/${lang}/manifests - # In case you want to use all validated data - # lhotse prepare commonvoice \ - # --split validated \ - # --language $lang \ - # -j $nj $dl_dir/$release data/${lang}/manifests - # touch data/${lang}/manifests/.cv-${lang}.validated.done - - # In case you want to take the risk and use invalidated data - # lhotse prepare commonvoice \ - # --split invalidated \ - # --language $lang \ - # -j $nj $dl_dir/$release data/${lang}/manifests - # touch data/${lang}/manifests/.cv-${lang}.invalidated.done + if [ $use_validated = true ] && [ ! -f data/${lang}/manifests/.cv-${lang}.validated.done ]; then + log "Also prepare validated data" + lhotse prepare commonvoice \ + --split validated \ + --language $lang \ + -j $nj $dl_dir/$release data/${lang}/manifests + touch data/${lang}/manifests/.cv-${lang}.validated.done + fi + + if [ $use_invalidated = true ] && [ ! -f data/${lang}/manifests/.cv-${lang}.invalidated.done ]; then + log "Also prepare invalidated data" + lhotse prepare commonvoice \ + --split invalidated \ + --language $lang \ + -j $nj $dl_dir/$release data/${lang}/manifests + touch data/${lang}/manifests/.cv-${lang}.invalidated.done + fi touch data/${lang}/manifests/.cv-${lang}.done fi @@ -123,7 +133,8 @@ if [ $stage -le 1 ] && [ $stop_stage -ge 1 ]; then # 1. wget -O jq https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 # 2. chmod +x ./jq # 3. cp jq /usr/bin - if [ -f data/${lang}/manifests/.cv-${lang}.validated.done ]; then + if [ $use_validated = true ]; then + log "Getting cut ids from dev/test sets for later use" gunzip -c data/${lang}/manifests/cv-${lang}_supervisions_test.jsonl.gz \ | jq '.id' | sed 's/"//g' > data/${lang}/manifests/cv-${lang}_test_ids @@ -148,18 +159,18 @@ if [ $stage -le 3 ] && [ $stop_stage -ge 3 ]; then if [ ! -e data/${lang}/fbank/.preprocess_complete ]; then ./local/preprocess_commonvoice.py --language $lang touch data/${lang}/fbank/.preprocess_complete + fi + + if [ $use_validated = true ] && [ ! -f data/${lang}/fbank/.validated.preprocess_complete ]; then + log "Also preprocess validated data" + ./local/preprocess_commonvoice.py --language $lang --split validated + touch data/${lang}/fbank/.validated.preprocess_complete + fi - if [ -f data/${lang}/manifests/.cv-${lang}.validated.done ]; then - log "Also preprocess validated data" - ./local/preprocess_commonvoice.py --language $lang --split validated - touch data/${lang}/fbank/.validated.preprocess_complete - fi - - if [ -f data/${lang}/manifests/.cv-${lang}.invalidated.done ]; then - log "Also preprocess invalidated data" - ./local/preprocess_commonvoice.py --language $lang --split invalidated - touch data/${lang}/fbank/.invalidated.preprocess_complete - fi + if [ $use_invalidated = true ] && [ ! -f data/${lang}/fbank/.invalidated.preprocess_complete ]; then + log "Also preprocess invalidated data" + ./local/preprocess_commonvoice.py --language $lang --split invalidated + touch data/${lang}/fbank/.invalidated.preprocess_complete fi fi @@ -180,16 +191,16 @@ if [ $stage -le 5 ] && [ $stop_stage -ge 5 ]; then touch $split_dir/.cv-${lang}_train_split.done fi - if [ -f data/${lang}/fbank/.validated.preprocess_complete ]; then + split_dir=data/${lang}/fbank/cv-${lang}_validated_split_${num_splits} + if [ $use_validated = true ] && [ ! -f $split_dir/.cv-${lang}_validated.done ]; then log "Also split validated data" - split_dir=data/${lang}/fbank/cv-${lang}_validated_split_${num_splits} lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_validated_raw.jsonl.gz $split_dir touch $split_dir/.cv-${lang}_validated.done fi - if [ -f data/${lang}/fbank/.invalidated.preprocess_complete ]; then + split_dir=data/${lang}/fbank/cv-${lang}_invalidated_split_${num_splits} + if [ $use_invalidated = true ] && [ ! -f $split_dir/.cv-${lang}_invalidated.done ]; then log "Also split invalidated data" - split_dir=data/${lang}/fbank/cv-${lang}_invalidated_split_${num_splits} lhotse split $num_splits ./data/${lang}/fbank/cv-${lang}_cuts_invalidated_raw.jsonl.gz $split_dir touch $split_dir/.cv-${lang}_invalidated.done fi @@ -208,8 +219,7 @@ if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then touch data/${lang}/fbank/.cv-${lang}_train.done fi - split_dir=data/${lang}/fbank/cv-${lang}_validated_split_${num_splits} - if [ -f $split_dir/.cv-${lang}_validated.done ]; then + if [ $use_validated = true ] && [ ! -f data/${lang}/fbank/.cv-${lang}_validated.done ]; then log "Also compute features for validated data" ./local/compute_fbank_commonvoice_splits.py \ --subset validated \ @@ -222,8 +232,7 @@ if [ $stage -le 6 ] && [ $stop_stage -ge 6 ]; then touch data/${lang}/fbank/.cv-${lang}_validated.done fi - split_dir=data/${lang}/fbank/cv-${lang}_invalidated_split_${num_splits} - if [ -f $split_dir/.cv-${lang}_invalidated.done ]; then + if [ $use_invalidated = true ] && [ ! -f data/${lang}/fbank/.cv-${lang}_invalidated.done ]; then log "Also compute features for invalidated data" ./local/compute_fbank_commonvoice_splits.py \ --subset invalidated \ @@ -244,13 +253,13 @@ if [ $stage -le 7 ] && [ $stop_stage -ge 7 ]; then lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_train.jsonl.gz fi - if [ -f data/${lang}/fbank/.cv-${lang}_validated.done ]; then + if [ $use_validated = true ] && [ -f data/${lang}/fbank/.cv-${lang}_validated.done ]; then log "Also combine features for validated data" pieces=$(find data/${lang}/fbank/cv-${lang}_validated_split_${num_splits} -name "cv-${lang}_cuts_validated.*.jsonl.gz") lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_validated.jsonl.gz fi - if [ -f data/${lang}/fbank/.cv-${lang}_invalidated.done ]; then + if [ $use_invalidated = true ] && [ -f data/${lang}/fbank/.cv-${lang}_invalidated.done ]; then log "Also combine features for invalidated data" pieces=$(find data/${lang}/fbank/cv-${lang}_inalidated_split_${num_splits} -name "cv-${lang}_cuts_invalidated.*.jsonl.gz") lhotse combine $pieces data/${lang}/fbank/cv-${lang}_cuts_invalidated.jsonl.gz