From 9797e558bf6fb5862b6de4bc4da637d9d7cadc44 Mon Sep 17 00:00:00 2001 From: chaklim Date: Thu, 12 Dec 2019 13:16:14 +0800 Subject: [PATCH] Update TRAIN.md --- docs/TRAIN.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/TRAIN.md b/docs/TRAIN.md index 1f7cb59..88820f8 100644 --- a/docs/TRAIN.md +++ b/docs/TRAIN.md @@ -201,7 +201,7 @@ parallel --joblog ./uncompress_tensors.log -j${THREADS_LOW} -N2 \ --line-buffer --shuf --verbose --compress stdbuf -i0 -o0 -e0 pigz -p4 -dc ::: | \ parallel --joblog ./round_robin_cat.log -j${THREADS} \ --line-buffer --pipe -N1000 --no-keep-order --round-robin --compress \ -"split - -l ${ESTIMATED_SPLIT_NO_OF_LINES} --filter='shuf | pigz -p4 > \$FILE.gz' -d ${SHUFFLED_TENSORS_FILE_PATH}/split_{#}_" +"split - -l ${ESTIMATED_SPLIT_NO_OF_LINES} --filter='shuf | pigz -p4 > \$FILE.gz' -d ${SHUFFLED_TENSORS_FOLDER_PATH}/split_{#}_" ``` ##### Two rounds shuffling (paranoid, but used in paper) @@ -212,15 +212,15 @@ parallel --joblog ./uncompress_tensors_round_1.log -j${THREADS_LOW} -N2 \ --line-buffer --shuf --verbose --compress stdbuf -i0 -o0 -e0 pigz -p4 -dc ::: | \ parallel --joblog ./round_robin_cat_round_1.log -j${THREADS} \ --line-buffer --pipe -N1000 --no-keep-order --round-robin --compress \ -"split - -l ${ESTIMATED_SPLIT_NO_OF_LINES} --filter='shuf | pigz -p4 > \$FILE.gz' -d ${SHUFFLED_TENSORS_FILE_PATH}/round1_{#}_" +"split - -l ${ESTIMATED_SPLIT_NO_OF_LINES} --filter='shuf | pigz -p4 > \$FILE.gz' -d ${SHUFFLED_TENSORS_FOLDER_PATH}/round1_{#}_" # the second round -ls ${SHUFFLED_TENSORS_FILE_PATH}/round1_* | \ +ls ${SHUFFLED_TENSORS_FOLDER_PATH}/round1_* | \ parallel --joblog ./uncompress_tensors_round_1.log -j${THREADS_LOW} -N2 \ --line-buffer --shuf --verbose --compress stdbuf -i0 -o0 -e0 pigz -p4 -dc ::: | \ parallel --joblog ./round_robin_cat.log -j${THREADS} \ --line-buffer --pipe -N1000 --no-keep-order --round-robin --compress \ -"split - -l ${ESTIMATED_SPLIT_NO_OF_LINES} --filter='shuf | pigz -p4 > \$FILE.gz' -d ${SHUFFLED_TENSORS_FILE_PATH}/split_{#}_" +"split - -l ${ESTIMATED_SPLIT_NO_OF_LINES} --filter='shuf | pigz -p4 > \$FILE.gz' -d ${SHUFFLED_TENSORS_FOLDER_PATH}/split_{#}_" ``` #### 9. Create splited binaries using the `Tensor2Bin` submodule @@ -426,7 +426,7 @@ parallel --joblog ./uncompress_tensors.log -j${THREADS_LOW} -N2 \ --line-buffer --shuf --verbose --compress stdbuf -i0 -o0 -e0 pigz -p4 -dc ::: | \ parallel --joblog ./round_robin_cat.log -j${THREADS} \ --line-buffer --pipe -N1000 --no-keep-order --round-robin --compress \ -"split - -l ${ESTIMATED_SPLIT_NO_OF_LINES} --filter='shuf | pigz -p4 > \$FILE.gz' -d ${SHUFFLED_TENSORS_FILE_PATH}/split_{#}_" +"split - -l ${ESTIMATED_SPLIT_NO_OF_LINES} --filter='shuf | pigz -p4 > \$FILE.gz' -d ${SHUFFLED_TENSORS_FOLDER_PATH}/split_{#}_" ``` #### 8. Create splited binaries using the `Tensor2Bin` submodule