show hashing rate from the start

watch_status

@@ -8,6 +8,7 @@
 trap - EXIT
 trap 'exit 0' SIGUSR1
 
+startTime=$(date +%s)
 minWait=10
 maxWait=600
 export LC_ALL=C # faster grep
@@ -16,60 +17,82 @@ hashFile="$1"
 if [[ ! -z "$SLURM_JOB_ID" ]]; then
     printf 'Starting progress monitoring at %s for %s.\n' "$(date)" "$hashFile"
     printf 'Direcories: %s\n' "$(printf "'%s' " "${@:2}")"
-    printf 'Calculating total size...\n'
+    printf 'Calculating total size in background...\n'
     cd "$SLURM_SUBMIT_DIR"
 fi
 
-bytesHashed(){
-    cat <(grep -va "^#\|^%" "$hashFile" 2> /dev/null | cut -d, -f1) \
-        <(printf 0) | paste -sd+ | bc
+getState(){
+    exec 10<> <(:)
+    bytes=$(cat <(grep -va "^#\|^%" "$hashFile" 2> /dev/null |
+            cut -d, -f1 | tee >(wc -l >&10) ) \
+            <(printf 0) | paste -sd+ | bc)
+    read -u 10 nfiles
+    exec 10>&-
+    printf '%d %d' $bytes $nfiles
 }
 hashStartTime=$(date +%s)
-hashStartState=$(bytesHashed)
+read hashStartState startFilesCount <<< $(getState)
+
+lastFile(){
+    tail -n1 "$hashFile" | cut -d, -f3
+}
 
 sleepUntil(){
     local now=$(date +%s)
     (( $1 > $now )) && sleep $(( $1 - $now ))
 }
 
-checkpoint=$(date +%s)
-total=$(du -sb --apparent-size -- "${@:2}" | \
-    cut -f1 | sed '/[^0-9]/d' | paste -sd+ | bc)
-TOTAL="$(numfmt --to=iec-i $total)B"
-waitTime=$(( $(date +%s) - $checkpoint ))
-waitTime=$(( $waitTime>$minWait?$waitTime:$minWait ))
-waitTime=$(( $waitTime<$maxWait?$waitTime:$maxWait ))
+total_file=$(mktemp)
+set +m
+{ du -sb --apparent-size -- "${@:2}" | \
+    cut -f1 | sed '/[^0-9]/d' | paste -sd+ | \
+    bc > $total_file & } 2>/dev/null
+duPID=$!
+trap 'kill $duPID 2> /dev/null; rm $total_file' EXIT
+
+TOTAL='?B'
 
 displayTime(){
   local T=$1
   local D=$((T/60/60/24))
   local H=$((T/60/60%24))
   local M=$((T/60%60))
   local S=$((T%60))
-  (( $D > 0 )) && printf '%d days ' $D
-  (( $H > 0 )) && printf '%d h ' $H
-  (( $M > 0 )) && printf '%d min ' $M
-  (( $D > 0 || $H > 0 || $M > 0 )) && printf 'and '
+  (( $D != 0 )) && printf '%d days ' $D
+  (( $H != 0 )) && printf '%d h ' $H
+  (( $M != 0 )) && printf '%d min ' $M
+  (( $D != 0 || $H > 0 || $M > 0 )) && printf 'and '
   printf '%d sec\n' $S
 }
-# when `du` is quicker than the output of the super script:
-sleepUntil $(( $checkpoint + 1 ))
-printf 'Updating hashing progress every %s...\n' "$(displayTime $waitTime)"
 
 finished(){
     printf '[%(%F %H:%M:%S)T] 100.00 %% (%s/%s)\n' \
         "$(date +%s)" "$TOTAL" "$TOTAL"
     exit 0
 }
 trap 'finished; exit 0' SIGUSR1
+
 lastTime=$hashStartTime
 lastState=$hashStartState
+lastFileCount=$startFilesCount
+STATE=' ?%'
+# wait for output of the super script:
+sleepUntil $(( $startTime + 1 ))
 while true; do
     checkpoint=$(date +%s)
-    current=$(bytesHashed)
+    read current filecount <<< $(getState)
+    if [ -z "$total" ]; then
+        total="$(cat $total_file)"
+        [ -z "$total" ] || TOTAL="$(numfmt --to=iec-i $total)B"
+        waitTime=$(( $checkpoint - $startTime ))
+        waitTime=$(( $waitTime>$minWait?$waitTime:$minWait ))
+        waitTime=$(( $waitTime<$maxWait?$waitTime:$maxWait ))
+    fi
+    if [ ! -z "$total" ]; then
+        printf -v state "%02d" $(( 10000 * $current / $total ))
+        STATE=" ${state:0:-2}.${state: -2} %"
+    fi
     CURRENT=" ($(numfmt --to=iec-i $current)B/$TOTAL)"
-    printf -v state "%02d" $(( 10000 * $current / $total ))
-    STATE=" ${state:0:-2}.${state: -2} %"
     time=$(date "+%F %H:%M:%S")
     if [[ $current -le $hashStartState ]]; then
         hashStartState=$current
@@ -80,15 +103,21 @@ while true; do
         passed=$(( $now - $lastTime ))
         lastTime=$now
         diff=$(( $current - $lastState ))
+        diffF=$(( $filecount - $lastFileCount ))
         lastState=$current
+        lastFileCount=$filecount
         speed=$(( $diff / $passed ))
-        SPEED=" at $(numfmt --to=iec-i $speed)B/s"
-        total_passed=$(( $(date +%s) - $hashStartTime ))
+        speedF=$(bc <<< "scale=2; 60 * $diffF / $passed")
+        SPEED=" at $(numfmt --to=iec-i $speed)B/s and $speedF files/min"
+        total_passed=$(( $now - $hashStartTime ))
         total_diff=$(( $current - $hashStartState ))
-        rem=$(( $total - $current ))
-        eta=$(( $rem * $total_passed / $total_diff ))
-        ETA=" ~ $(displayTime $eta) remaining"
+        if [ ! -z "$total" ]; then
+            rem=$(( $total - $current ))
+            eta=$(bc <<< "$rem * $total_passed / $total_diff")
+            ETA=" ~ $(displayTime $eta) remaining"
+        fi
     fi
-    printf '[%s]%s%s%s%s\n' "$time" "$STATE" "$CURRENT" "$SPEED" "$ETA"
+    LAST=" ($(lastFile))"
+    printf '[%s]%s%s%s%s%s\n' "$time" "$STATE" "$CURRENT" "$SPEED" "$ETA" "$LAST"
     sleepUntil $(( $checkpoint + $waitTime ))
 done