updates

CHANGELOG.md

@@ -10,7 +10,7 @@
 
 - adding `requirements.txt` for easy creation of environment in "spacesavers2" docker (#68, @kopardev)
 - `grubbers` has new `--outfile` argument.
-- `blamematrix` has 3 new arguments `--humanreable`, `--includezeros` and `--outfile`.
+- `blamematrix` has now been moved into `mimeo`.
 - `mimeo` files.gz always includes the original file as the first one in the filelist.
 - `mimeo` now has kronatools compatible output. ktImportText is also run if in PATH to generate HTML report for duplicates only. (#46, @kopardev)
 - documentation updated.
@@ -23,6 +23,8 @@
 - `blamematrix` fixed to account for changes due to #71
 - `usurp` fixed to account for changes due to #71. Now using the new "original file" column while creating hard-links.
 - `e2e` overhauled, improved and well commented.
+- total size now closely resemble `df` results (fix #75 @kopardev)
+- files with future timestamps are handles correctly (fix #76, @kopardev)
 
 ## spacesavers2 0.10.2
 

README.md

@@ -20,19 +20,13 @@ Welcome! `spacesavers2`:
 
 > New improved parallel implementation of [`spacesavers`](https://github.com/CCBR/spacesavers). `spacesavers` is soon to be decommissioned!
 
-> Note: `spacesavers2` requires [python version 3.11](https://www.python.org/downloads/release/python-3110/) or later and the [xxhash](https://pypi.org/project/xxhash/) library. These dependencies are already installed on biowulf (as a conda env). The environment for running `spacesavers2` can get set up using:
->
-> ```bash
-> . "/data/CCBR_Pipeliner/db/PipeDB/Conda/etc/profile.d/conda.sh" && \
-> conda activate py311
-> ```
+> Note: `spacesavers2` requires [python version 3.11](https://www.python.org/downloads/release/python-3110/) or later and the [xxhash](https://pypi.org/project/xxhash/) library. These dependencies are already installed on biowulf (as a conda env).
 
 ## `spacesavers2` has the following Basic commands:
 
 - spacesavers2_catalog
 - spacesavers2_mimeo
 - spacesavers2_grubbers
-- spacesavers2_blamematrix
 - spacesavers2_e2e
 - spacesavers2_usurp
 

mkdocs.yml

@@ -102,6 +102,5 @@ nav:
       - catalog: catalog.md
       - mimeo: mimeo.md
       - grubbers: grubbers.md
-      - blamematrix: blamematrix.md
       - usurp: usurp.md
       - e2e: e2e.md

spacesavers2_catalog

@@ -17,19 +17,17 @@ from pathlib import Path
 
 
 def task(f):
-    if not os.path.isfile(f):
-        return ""
-    else:
-        fd = FileDetails()
-        fd.initialize(
-            f,
-            buffersize=args.buffersize,
-            thresholdsize=args.ignoreheadersize,
-            tb=args.buffersize,
-            sed=sed,
-            bottomhash=args.bottomhash,
-        )
-        return "%s" % (fd)
+    fd = FileDetails()
+    fd.initialize(
+        f,
+        buffersize=args.buffersize,
+        thresholdsize=args.ignoreheadersize,
+        tb=args.buffersize,
+        sed=sed,
+        bottomhash=args.bottomhash,
+        st_block_byte_size=args.st_block_byte_size,
+    )
+    return "%s" % (fd)
 
 
 def main():
@@ -84,14 +82,23 @@ def main():
         help="this sized header of the file is ignored before extracting buffer of buffersize for xhash creation (only for special extensions files) default = 1024 * 1024 * 1024 bytes",
     )
     parser.add_argument(
-        "-s",
+        "-x",
         "--se",
         dest="se",
         required=False,
         type=str,
         default="bam,bai,bigwig,bw,csi",
         help="comma separated list of special extensions (default=bam,bai,bigwig,bw,csi)",
     )
+    parser.add_argument(
+        "-s",
+        "--st_block_byte_size",
+        dest="st_block_byte_size",
+        required=False,
+        default=512,
+        type=int,
+        help="st_block_byte_size on current filesystem (default 512)",
+    )
     parser.add_argument(
         "-o",
         "--outfile",
@@ -120,7 +127,9 @@ def main():
 
     folder = args.folder
     p = Path(folder)
-    files = p.glob("**/*")
+    files = [p]
+    files2 = p.glob("**/*")
+    files.extend(files2)
 
     if args.outfile:
         outfh = open(args.outfile, "w")

spacesavers2_e2e

@@ -13,8 +13,8 @@ source ${SCRIPT_DIR}/resources/argparse.bash || exit 1
 argparse "$@" <<EOF || exit 1
 parser.add_argument('-f','--folder',required=True, help='Folder to run spacesavers_catalog on.')
 parser.add_argument('-p','--threads',required=False, help='number of threads to use', default=4)
+parser.add_argument('-d','--maxdepth',required=False, help='maxdepth for mimeo', default=4)
 parser.add_argument('-l','--limit',required=False, help='limit for running spacesavers_grubbers', default=5)
-parser.add_argument('-v','--level',required=False, help='level for running spacesavers_blamematrix', default=3)
 parser.add_argument('-q','--quota',required=False, help='total size of the volume (default = 200 for /data/CCBR)', default=200)
 parser.add_argument('-o','--outfolder',required=True, help='Folder where all spacesavers_e2e output files will be saved')
 EOF
@@ -54,21 +54,25 @@ spacesavers2_catalog \
     > ${outfile_catalog_log} 2> ${outfile_catalog_err}
 fi
 
+sleep 60
+
 # spacesavers2_mimeo
 if [ "$?" == "0" ];then
 echo "Running spacesavers2_mimeo" 
-command -V ktImportText 2>/dev/null || module load kronatools || (>&2 echo "module kronatools could not be loaded"; exit 1)
+command -V ktImportText 2>/dev/null || module load kronatools || (>&2 echo "module kronatools could not be loaded")
 spacesavers2_mimeo \
     --catalog ${outfile_catalog} \
     --outdir ${OUTFOLDER} \
     --quota $QUOTA \
     --duplicatesonly \
-    --maxdepth 3 \
+    --maxdepth $MAXDEPTH \
     --p $prefix \
     --kronaplot \
     > ${outfile_mimeo_log} 2> ${outfile_mimeo_err}
 fi
 
+sleep 60
+
 # spacesavers2_grubbers
 if [ "$?" == "0" ];then
 echo "Running spacesavers2_grubbers" && \
@@ -84,13 +88,4 @@ for filegz in `ls ${OUTFOLDER}/${prefix}*files.gz`;do
 done
 fi
 
-# spacesavers2_blamematrix
-if [ "$?" == "0" ];then
-echo "Running spacesavers2_blamematrix" && \
-spacesavers2_blamematrix \
-    --filesgz ${OUTFOLDER}/${prefix}.allusers.mimeo.files.gz \
-    --level $LEVEL \
-    --outfile ${outfile_blamematrix} \
-    > ${outfile_blamematrix_log} 2> ${outfile_blamematrix_err}
-fi
-echo "Done!"
+echo "Done!"

spacesavers2_grubbers

@@ -92,19 +92,19 @@ def main():
         of = sys.stdout
 
     for fgitem in dups:
-        if fgitem.totalsize < top_limit:
+        if fgitem.totalsize <= top_limit:
             break
         saved += fgitem.totalsize
         of.write("%s\n"%(fgitem))
 
     if args.outfile:
         of.close()
 
-    saved = get_human_readable_size(saved)
+    hrsaved = get_human_readable_size(saved)
     print_with_timestamp(
         start=start,
         scriptname=scriptname,
-        string="Deleting top grubbers will save {}!".format(saved),
+        string="Deleting top grubbers will save {} [ {} Bytes ] !".format(hrsaved,saved),
     )
     print_with_timestamp(start=start, scriptname=scriptname, string="Done!")