update slow5lib and --index option for get

hasindu2008 · Aug 12, 2023 · c876f93 · c876f93
1 parent 7663891
commit c876f93
Show file tree

Hide file tree

Showing 24 changed files with 538 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -14,7 +14,7 @@ Full documentation: https://hasindu2008.github.io/slow5tools<br/>
 Publication (SLOW5 format): https://www.nature.com/articles/s41587-021-01147-4<br/>
 Publication (slow5tools): https://genomebiology.biomedcentral.com/articles/10.1186/s13059-023-02910-3<br/>
 SLOW5 specification: https://hasindu2008.github.io/slow5specs<br/>
-slow5lib: https://github.com/hasindu2008/slow5lib<br/>
+slow5 ecosystem: https://hasindu2008.github.io/slow5<br/>
 
 To convert to and from ONT's new POD5 format, you use [blue_crab](https://github.com/Psy-Fer/blue-crab). If POD5 format and the associated POD5 C/C++ API reaches maturity/stability and adheres to C++11 standard, capabilities for POD5 <-> SLOW5 conversion will be added to slow5tools. slow5tools is strictly adhering to C++11 standard for wider compatibility.
 

diff --git a/docs/commands.md b/docs/commands.md
@@ -170,6 +170,8 @@ slow5tools get [OPTIONS] file1.blow5 --list readids.txt
     The batch size. This is the number of records on the memory at once [default value: 4096]. An increased batch size improves multi-threaded performance at cost of higher RAM.
 * `-l, --list FILE`:<br/>
     List of read ids provided as a single-column text file with one read id per line.
+* `--index FILE`:<br/>
+    Path to a custom slow5 index (experimental). Useful if your index file is located somewhere other than in the same directory as the input S/BLOW5 file.
 *  `-h`, `--help`:<br/>
     Prints the help menu.
 

diff --git a/docs/getting_started.md b/docs/getting_started.md
@@ -10,7 +10,7 @@ SLOW5 is a new file format for storing signal data from Oxford Nanopore Technolo
 Publication (SLOW5 format): https://www.nature.com/articles/s41587-021-01147-4<br/>
 Publication (slow5tools): https://genomebiology.biomedcentral.com/articles/10.1186/s13059-023-02910-3<br/>
 SLOW5 specification: https://hasindu2008.github.io/slow5specs<br/>
-slow5lib: https://github.com/hasindu2008/slow5lib<br/>
+slow5 ecosystem: https://hasindu2008.github.io/slow5<br/>
 
 To convert to and from ONT's new POD5 format, you use [blue_crab](https://github.com/Psy-Fer/blue-crab). If POD5 format and the associated POD5 C/C++ API reaches maturity/stability and adheres to C++11 standard, capabilities for POD5 <-> SLOW5 conversion will be added to slow5tools. slow5tools is strictly adhering to C++11 standard for wider compatibility.
 

diff --git a/slow5lib b/slow5lib
diff --git a/src/get.c b/src/get.c
@@ -28,6 +28,7 @@
     HELP_MSG_BATCH \
     "    -l --list [FILE]              list of read ids provided as a single-column text file with one read id per line.\n" \
     "    --skip                        warn and continue if a read_id was not found.\n" \
+    "    --index [FILE]                path to a custom slow5 index (experimental).\n" \
     HELP_MSG_HELP \
     HELP_FORMATS_METHODS
 
@@ -117,6 +118,7 @@ int get_main(int argc, char **argv, struct program_meta *meta) {
         {"threads",     required_argument, NULL, 't' }, //7
         {"help",        no_argument, NULL, 'h' }, //8
         {"benchmark",   no_argument, NULL, 'e' }, //9
+        {"index",       required_argument, NULL, 0 }, //10
         {NULL, 0, NULL, 0 }
     };
 
@@ -128,6 +130,7 @@ int get_main(int argc, char **argv, struct program_meta *meta) {
 
     // Input arguments
     char* read_list_file_in = NULL;
+    const char *slow5_index = NULL;
 
     int opt;
     int longindex = 0;
@@ -173,8 +176,12 @@ int get_main(int argc, char **argv, struct program_meta *meta) {
                     case 6:
                         skip_flag = 1;
                         break;
+                    case 10:
+                        slow5_index = optarg;
+                        break;
                 }
                 break;
+
             default: // case '?'
                 fprintf(stderr, HELP_SMALL_MSG, argv[0]);
                 EXIT_MSG(EXIT_FAILURE, argv, meta);
@@ -274,11 +281,21 @@ int get_main(int argc, char **argv, struct program_meta *meta) {
         }
     }
 
-    int ret_idx = slow5_idx_load(slow5file);
-    if (ret_idx < 0) {
-        ERROR("Error loading index file for %s\n", f_in_name);
-        EXIT_MSG(EXIT_FAILURE, argv, meta);
-        return EXIT_FAILURE;
+    if(slow5_index == NULL){
+        int ret_idx = slow5_idx_load(slow5file);
+        if (ret_idx < 0) {
+            ERROR("Error loading index file for %s\n", f_in_name);
+            EXIT_MSG(EXIT_FAILURE, argv, meta);
+            return EXIT_FAILURE;
+        }
+    } else {
+        WARNING("%s","Loading index from custom path is an experimental feature. keep an eye.");
+        int ret_idx = slow5_idx_load_with(slow5file, slow5_index);
+        if (ret_idx < 0) {
+            ERROR("Error loading index file for %s from file path %s\n", f_in_name, slow5_index);
+            EXIT_MSG(EXIT_FAILURE, argv, meta);
+            return EXIT_FAILURE;
+        }
     }
 
     if (read_stdin) {

diff --git a/test/test.sh b/test/test.sh
@@ -1,5 +1,31 @@
 #!/bin/sh
 
+# MIT License
+
+# Copyright (c) 2020 Hiruna Samarakoon
+# Copyright (c) 2020 Sasha Jenner
+# Copyright (c) 2020,2023 Hasindu Gamaarachchi
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+###############################################################################
+
 FIRST_FAILED_SET_OF_TESTCASES="NOT SET"
 FLAG_FIRST_FAILED_SET_OF_TESTCASES=0
 

diff --git a/test/test_all.sh b/test/test_all.sh
@@ -1,6 +1,28 @@
 #!/bin/bash
 
-# @author: Hiruna Samarakoon ([email protected])
+# MIT License
+
+# Copyright (c) 2020 Hiruna Samarakoon
+# Copyright (c) 2020 Sasha Jenner
+# Copyright (c) 2020,2023 Hasindu Gamaarachchi
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
 
 ###############################################################################
 

diff --git a/test/test_cat.sh b/test/test_cat.sh
@@ -1,5 +1,31 @@
 #!/bin/bash
 
+# MIT License
+
+# Copyright (c) 2020 Hiruna Samarakoon
+# Copyright (c) 2020 Sasha Jenner
+# Copyright (c) 2020,2023 Hasindu Gamaarachchi
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+###############################################################################
+
 # steps
 # cat slow5 files
 # cat blow5 files
@@ -11,7 +37,7 @@ RED='\033[0;31m' ; GREEN='\033[0;32m' ; NC='\033[0m' # No Color
 die() { echo -e "${RED}$1${NC}" >&2 ; echo ; exit 1 ; } # terminate script
 info() {  echo ; echo -e "${GREEN}$1${NC}" >&2 ; }
 # Relative path to "slow5/tests/"
-REL_PATH="$(dirname $0)/" 
+REL_PATH="$(dirname $0)/"
 
 #...directories files tools arguments commands clean
 OUTPUT_DIR="$REL_PATH/data/out/cat"

diff --git a/test/test_extensive.sh b/test/test_extensive.sh
@@ -1,5 +1,29 @@
 #!/bin/bash
 
+# MIT License
+
+# Copyright (c) 2020 Hiruna Samarakoon
+# Copyright (c) 2020 Sasha Jenner
+# Copyright (c) 2020,2023 Hasindu Gamaarachchi
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
 ###############################################################################
 
 Usage="test_extensive.sh"

diff --git a/test/test_f2s.sh b/test/test_f2s.sh
@@ -1,4 +1,31 @@
 #!/bin/bash
+
+# MIT License
+
+# Copyright (c) 2020 Hiruna Samarakoon
+# Copyright (c) 2020 Sasha Jenner
+# Copyright (c) 2020,2023 Hasindu Gamaarachchi
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+###############################################################################
+
 # Run f2s with different file, input and output formats.
 Usage="test_s2f.sh"
 

diff --git a/test/test_f2s_s2f_integrity.sh b/test/test_f2s_s2f_integrity.sh
@@ -1,4 +1,31 @@
 #!/bin/bash
+
+# MIT License
+
+# Copyright (c) 2020 Hiruna Samarakoon
+# Copyright (c) 2020 Sasha Jenner
+# Copyright (c) 2020,2023 Hasindu Gamaarachchi
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+###############################################################################
+
 # Run f2s, s2f, and again f2s and check if first produced slow5s are same as the last set.
 Usage1="f2s_s2f_integrity_test.sh"
 Usage2="f2s_s2f_integrity_test.sh [path to fast5 directory] [path to create a temporary directory][-c or --to (optional) [clean_fscache -f (optional)]"

diff --git a/test/test_f2s_view_diff.sh b/test/test_f2s_view_diff.sh
@@ -1,5 +1,31 @@
 #!/bin/bash
 
+# MIT License
+
+# Copyright (c) 2020 Hiruna Samarakoon
+# Copyright (c) 2020 Sasha Jenner
+# Copyright (c) 2020,2023 Hasindu Gamaarachchi
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+###############################################################################
+
 # steps
 # f2s fast5 to blow5
 # view blow5 to slow5

diff --git a/test/test_get.sh b/test/test_get.sh
@@ -1,4 +1,31 @@
 #!/bin/bash
+
+# MIT License
+
+# Copyright (c) 2020 Hiruna Samarakoon
+# Copyright (c) 2020 Sasha Jenner
+# Copyright (c) 2020,2023 Hasindu Gamaarachchi
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+###############################################################################
+
 # Run f2s with different file, input and output formats.
 Usage="get_test.sh"
 
@@ -131,5 +158,17 @@ $SLOW5_EXEC get "$RAW_DIR/example2.slow5" --skip --list "$RAW_DIR/list_with_inva
 slow5tools_quickcheck $OUTPUT_DIR
 info "testcase $TESTCASE passed"
 
+TESTCASE=9
+info "------------------- slow5tools get testcase $TESTCASE -------------------"
+$SLOW5_EXEC get "$RAW_DIR/example2.slow5" --index $RAW_DIR/example2.slow5.idx r1 r5 r3 --to blow5 -c zlib -s none > "$OUTPUT_DIR/extracted_reads2.blow5" || die "testcase $TESTCASE failed"
+slow5tools_quickcheck $OUTPUT_DIR
+diff -q "$EXP_DIR/expected_extracted_reads.blow5" "$OUTPUT_DIR/extracted_reads2.blow5" &>/dev/null
+if [ $? -ne 0 ]; then
+    info "${RED}ERROR: diff failed for 'slow5tools get testcase $TESTCASE'${NC}"
+    exit 1
+fi
+info "testcase $TESTCASE passed"
+
+
 rm -r $OUTPUT_DIR || die "Removing $OUTPUT_DIR failed" 1>&3 2>&4
 exit 0
+2 −1		README.md
+2 −1		docs/getting_started.md
+2 −2		docs/slow5_api/low_level_api/slow5_idx_load_with.md
+2 −1		docs/slow5_api/slow5_low_level_api.md
+8 −1		src/slow5_idx.c