From 54f6a2725c3b0f26b047d7162d704f9e4d1b9c30 Mon Sep 17 00:00:00 2001 From: yhoogstrate Date: Fri, 30 Aug 2019 11:47:06 +0200 Subject: [PATCH 1/3] working prototype that does not work under root/sudo? --- CMakeLists.txt | 17 ++++++- src/fuse.cpp | 123 ++++++++++++++++++++++++++++----------------- src/main_mount.cpp | 13 +++++ 3 files changed, 107 insertions(+), 46 deletions(-) create mode 100644 src/main_mount.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a5f9b54..182cc5a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -110,9 +110,23 @@ add_executable(fastafs src/fuse.cpp src/lsfastafs.cpp ) - set_target_properties(fastafs PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${BUILD_DIR}") +# mount-only binary, without all the other stuff 'mount.fastafs' [for fstab] +add_executable(mount.fastafs + src/main_mount.cpp + src/fasta_to_fastafs.cpp + src/ucsc2bit_to_fastafs.cpp + src/fastafs.cpp + src/ucsc2bit.cpp + src/twobit_byte.cpp + src/database.cpp + src/utils.cpp + src/fuse.cpp + src/lsfastafs.cpp + ) +set_target_properties(mount.fastafs PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${BUILD_DIR}") + add_library(libfastafs SHARED src/fasta_to_fastafs.cpp src/ucsc2bit_to_fastafs.cpp @@ -166,6 +180,7 @@ add_test(test_utils "${BUILD_TEST_DIR}/test_utils") # The compiled binary, usually to: /usr/local/bin/fastafs install(TARGETS fastafs DESTINATION "bin") +install(TARGETS mount.fastafs DESTINATION "bin") install(TARGETS libfastafs LIBRARY DESTINATION "lib" PUBLIC_HEADER DESTINATION "include/libfastafs") # ---------------------------------------------------------------------- diff --git a/src/fuse.cpp b/src/fuse.cpp index a3d06345..53f1bf62 100644 --- a/src/fuse.cpp +++ b/src/fuse.cpp @@ -328,9 +328,11 @@ fuse_operations operations = { + void print_fuse_help() { - std::cout << "usage: fasfafs [options]\n"; + std::cout << "usage: fasfafs mount [options]\n"; + std::cout << "usage: mount.fastafs [options]\n"; std::cout << "\n"; std::cout << "general options:\n"; std::cout << " -2 --2bit virtualise a 2bit file rather than FASTAFS UID\n"; @@ -413,56 +415,90 @@ fuse_instance *parse_args(int argc, char **argv, char **argv_fuse) //printf("argc=%i",argc); argv_fuse[fi->argc_fuse++] = (char *) "fastafs"; // becomes fuse.fastafs - //fi->argv_fuse[0] = argv[0]; - //fi->argv_fuse[1] = argv[1]; - int i = 2; + char current_argument = '\0';// could be o for '-o', etc. + + int i = 1; + std::vector full_args = {}; while(i < argc) { printf("processing argv[%i] = '%s';\n", i, argv[i]); - if(i < argc - 3) { // all arguments that take 2 arguments "--p", "50" - if(strcmp(argv[i], "-p") == 0 or strcmp(argv[i], "--padding") == 0) { - try { - sscanf(argv[++i], "%u", &fi->padding); - } catch(std::exception const & e) { - std::cerr << "ERROR: invalid padding value, must be integer value ranging from 0 to max-int size\n"; - exit(1); - } - } else { // arguments that need to be send to fuse - //argv_test[fi->argc_fuse] = argv[i]; - argv_fuse[fi->argc_fuse++] = argv[i]; + + if(current_argument != '\0') { // parse the arguments' value + switch(current_argument) { + case 'p': + try { + sscanf(argv[i], "%u", &fi->padding); + } catch(std::exception const & e) { + std::cerr << "ERROR: invalid padding value, must be integer value ranging from 0 to max-int size\n"; + exit(1); + } + break; + default: + argv_fuse[fi->argc_fuse++] = argv[i]; // -o contents must be send to fuse + printf("Skipping parsing following argument: -%c = %s\n", current_argument, argv[i]); + break; } - } else if(i < argc - 2) { // all arguments that one argument "--lowercase" switches etc - if(strcmp(argv[i], "-2") == 0 or strcmp(argv[i], "--2bit") == 0) { + } + else if(argv[i][0] == '-') { + if(strlen(argv[i]) == 1) { + // error + } + else if(argv[i][1] == '2') { // implement flags fi->from_fastafs = false; - } else { // arguments that need to be send to fuse - //argv_test[fi->argc_fuse] = argv[i]; - argv_fuse[fi->argc_fuse++] = argv[i]; } - } else if(i == argc - 2) { // this refers to the fastafs object - if(fi->from_fastafs) { - database d = database(); - std::string fname = d.get(argv[i]); - - if(fname.size() == 0) { // invalid mount argument, don't bind fastafs object - print_fuse_help(); - exit(1); - } else { // valid fastafs and bind fastafs object - fi->f = new fastafs(std::string(argv[i])); - fi->f->load(fname); - - fi->cache = fi->f->init_ffs2f(fi->padding, true);// allow mixed case + else { // remaining are no flags but arguments with values that need to be parsed + current_argument = argv[i][1]; + printf(" -> parsing arg: %c\n", current_argument); + + if(argv[i][1] == 'o' or argv[i][1] == 'd' or argv[i][1] == 'f') { + argv_fuse[fi->argc_fuse++] = argv[i]; } - } else { - std::string basename = std::filesystem::path(std::string(argv[i])).filename(); - - fi->u2b = new ucsc2bit(basename);// useses basename as prefix for filenames to mount: hg19.2bit -> hg19.2bit.fa - fi->u2b->load(std::string(argv[i])); } - } else { // mountpoint - //argv_test[fi->argc_fuse] = argv[i]; - argv_fuse[fi->argc_fuse++] = argv[i]; } - + else { + full_args.push_back(i); + printf(" candidate of std-args of which last 2 are needed\n"); + } i++; + + } + + if(full_args.size() >= 2) { + int mount_target_arg = full_args[full_args.size() - 2 ]; // last two arguments are and , location to last 2 args not starting with --/- are in this vector + + if(fi->from_fastafs) { + database d = database(); + std::string fname = d.get(argv[mount_target_arg]); + + std::string name; // prefix name of mounted fasta dict 2bit and fai files + if(fname.size() == 0) { // invalid mount argument, don't bind fastafs object + fname = std::string(argv[mount_target_arg]); + name = std::filesystem::path(fname).filename(); + + // remove .fastafs suffix + size_t lastindex = name.find_last_of("."); + name = name.substr(0, lastindex); + + } + else { + name = std::string(argv[mount_target_arg]); + } + + printf("[%s : %s]\n", name.c_str(), fname.c_str()); + + fi->f = new fastafs(name); + fi->f->load(fname); + + fi->cache = fi->f->init_ffs2f(fi->padding, true);// allow mixed case + } else { + std::string basename = std::filesystem::path(std::string(argv[mount_target_arg])).filename(); + + fi->u2b = new ucsc2bit(basename);// useses basename as prefix for filenames to mount: hg19.2bit -> hg19.2bit.fa + fi->u2b->load(std::string(argv[mount_target_arg])); + } + + //argv_fuse[fi->argc_fuse++] = argv[mount_target_arg]; + argv_fuse[fi->argc_fuse++] = argv[mount_target_arg + 1]; + argv_fuse[fi->argc_fuse++] = "-f"; } return fi; @@ -504,6 +540,3 @@ void fuse(int argc, char *argv[]) //http://www.maastaar.net/fuse/linux/filesystem/c/2016/05/21/writing-a-simple-filesystem-using-fuse/ } - - - diff --git a/src/main_mount.cpp b/src/main_mount.cpp new file mode 100644 index 00000000..64da826f --- /dev/null +++ b/src/main_mount.cpp @@ -0,0 +1,13 @@ + +#include + +#include "config.hpp" +#include "fuse.hpp" + +int main(int argc, char *argv[]) +{ + fuse(argc, argv); + + return 0; +} + From 755faf0e91d49e9e16d83ff0bbf407ca44f67a2f Mon Sep 17 00:00:00 2001 From: yhoogstrate Date: Fri, 30 Aug 2019 12:03:33 +0200 Subject: [PATCH 2/3] right order of passing arguments to fuse, only sudo and permissions is complicated still --- src/fuse.cpp | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/fuse.cpp b/src/fuse.cpp index 53f1bf62..b3e5cc13 100644 --- a/src/fuse.cpp +++ b/src/fuse.cpp @@ -412,8 +412,10 @@ fuse_instance *parse_args(int argc, char **argv, char **argv_fuse) fuse_instance *fi = new fuse_instance({nullptr, nullptr, true, nullptr, 60, 0}); - //printf("argc=%i",argc); + //fuse option variable to send to fuse argv_fuse[fi->argc_fuse++] = (char *) "fastafs"; // becomes fuse.fastafs + + std::vector fuse_options = {}; // those that need to be appended later char current_argument = '\0';// could be o for '-o', etc. @@ -433,7 +435,8 @@ fuse_instance *parse_args(int argc, char **argv, char **argv_fuse) } break; default: - argv_fuse[fi->argc_fuse++] = argv[i]; // -o contents must be send to fuse + //argv_fuse[fi->argc_fuse++] = argv[i]; // -o contents must be send to fuse + fuse_options.push_back(argv[i]); printf("Skipping parsing following argument: -%c = %s\n", current_argument, argv[i]); break; } @@ -450,7 +453,8 @@ fuse_instance *parse_args(int argc, char **argv, char **argv_fuse) printf(" -> parsing arg: %c\n", current_argument); if(argv[i][1] == 'o' or argv[i][1] == 'd' or argv[i][1] == 'f') { - argv_fuse[fi->argc_fuse++] = argv[i]; + //argv_fuse[fi->argc_fuse++] = argv[i]; + fuse_options.push_back(argv[i]); } } } @@ -486,9 +490,13 @@ fuse_instance *parse_args(int argc, char **argv, char **argv_fuse) printf("[%s : %s]\n", name.c_str(), fname.c_str()); fi->f = new fastafs(name); + printf(" new \n"); + fi->f->load(fname); + printf("loaded!\n"); fi->cache = fi->f->init_ffs2f(fi->padding, true);// allow mixed case + printf("initiated cache!\n"); } else { std::string basename = std::filesystem::path(std::string(argv[mount_target_arg])).filename(); @@ -498,7 +506,9 @@ fuse_instance *parse_args(int argc, char **argv, char **argv_fuse) //argv_fuse[fi->argc_fuse++] = argv[mount_target_arg]; argv_fuse[fi->argc_fuse++] = argv[mount_target_arg + 1]; - argv_fuse[fi->argc_fuse++] = "-f"; + for(size_t j = 0; j < fuse_options.size(); j++) { + argv_fuse[fi->argc_fuse++] = fuse_options[j]; + } } return fi; From 6e945933b8e6529929fc9908def27733bf04f6cf Mon Sep 17 00:00:00 2001 From: yhoogstrate Date: Fri, 6 Sep 2019 09:34:17 +0200 Subject: [PATCH 3/3] release notes 1.6.2 --- CMakeLists.txt | 2 +- Changelog | 7 ++++ README.md | 26 ++++++++++++++- src/fuse.cpp | 88 +++++++++++++++++++++++--------------------------- src/main.cpp | 7 +++- 5 files changed, 80 insertions(+), 50 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 182cc5a8..d561ee13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ project(fastafs) # Do this once in a while - find different bugs #set(CMAKE_CXX_COMPILER "clang++") -set(PROJECT_VERSION "1.6.1") +set(PROJECT_VERSION "1.6.2") set(PACKAGE_URL "https://github.com/yhoogstrate/fastafs") set(PACKAGE_BUGREPORT "${PACKAGE_URL}/issues") diff --git a/Changelog b/Changelog index 0f6d4ee6..7d6166b6 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,10 @@ +2019-09-06 Youri Hoogstrate + + * v1.6.2 + * Rewritten argument parsing of fuse/mounting + * Added binary `mount.fastafs` only for mounting, wihtout the toolkit + * `fastafs --version` will now also output whether the binary was compiled as debug or release. + 2019-08-21 Youri Hoogstrate * v1.6.1 diff --git a/README.md b/README.md index 73ed1301..5cff4e4b 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Here we propose a solution; a mixture of 2bit compression and random on FASTA ve FASTAFS is compatible with 2bit files but 2bit has some limitations and does for instance not allow mounting DICT files needed for CRAM and Picard tools. The 2bit files can be convert to FASTAFS and, more importantly, a mounted FASTAFS mountpoint will not only virtualize the FASTA but also the 2bit file. Is FASTAFS this famous 15th standard ()? Partially, but it is not designed to replace FASTA nor 2bit as those mountpoints should be used as flat files did before and the virtualization of dict files is not possible when we mount 2bit files. -## installation and compilation +## Installation and compilation Currently the package uses cmake for compilation Required dependencies are: @@ -165,6 +165,30 @@ aaaa c ``` +### Find all running `fastafs mount` / `mount.fastafs` instances + +The output format of `fastafs ps` is: `\t\t\n` + +``` +$ fastafs ps +16383 /home/youri/.local/share/fastafs/test.fastafs /mnt/tmp +``` + +### Mounting via fstab (for instance on linux boot) + +You can add the following line(s) to /etc/fstab to make fastafs mount during boot: + +``` +mount.fastafs#/home/youri/.local/share/fastafs/hg19.fastafs /mnt/fastafs/hg19 fuse auto,allow_other 0 0 +``` + +Here `mount.fastafs` refers to the binary that only does mounting, without the rest of the toolkit. +This is followed by a hash-tag and the path of the desired fastafs file. The next value is the mount point followed by the argument indicating it runs fuse. +The `auto,allow_other` refers to the `-o` argument. +Here `auto` ensures it is mounted automatically after boot. +Given that a system mounts as root user, the default permission of the mount point will be only for root. +By setting `allow_other`, file system users get permissions to the mountpoint. + ## Contributing Feel free to start a discussion or to contribute to the GPL licensed code. diff --git a/src/fuse.cpp b/src/fuse.cpp index b3e5cc13..228a0c43 100644 --- a/src/fuse.cpp +++ b/src/fuse.cpp @@ -230,8 +230,9 @@ static int do_getxattr(const char* path, const char* name, char* value, size_t s sprintf(mypid, "%d", getpid()); size_t pid_size = (size_t) strlen(mypid); + if(size > pid_size) { - strncpy(value, mypid, pid_size); + strcpy(value, mypid);// if statement makes strncpy unneeded value[pid_size] = '\0'; return (int) pid_size + 1; @@ -414,56 +415,57 @@ fuse_instance *parse_args(int argc, char **argv, char **argv_fuse) //fuse option variable to send to fuse argv_fuse[fi->argc_fuse++] = (char *) "fastafs"; // becomes fuse.fastafs - + std::vector fuse_options = {}; // those that need to be appended later char current_argument = '\0';// could be o for '-o', etc. - int i = 1; std::vector full_args = {}; - while(i < argc) { - printf("processing argv[%i] = '%s';\n", i, argv[i]); - + for(unsigned int i = 0; i < argc; ++i) { + printf("processing argv[%i] = '%s' [current argument=%i]\n", i, argv[i], (int) current_argument); + if(current_argument != '\0') { // parse the arguments' value switch(current_argument) { - case 'p': - try { - sscanf(argv[i], "%u", &fi->padding); - } catch(std::exception const & e) { - std::cerr << "ERROR: invalid padding value, must be integer value ranging from 0 to max-int size\n"; - exit(1); - } + case 'p': // scanning padding value + if(!sscanf(argv[i], "%u", &fi->padding)) { + throw std::invalid_argument("Invalid argument (padding value): " + std::string(argv[i])); + } break; - default: - //argv_fuse[fi->argc_fuse++] = argv[i]; // -o contents must be send to fuse - fuse_options.push_back(argv[i]); - printf("Skipping parsing following argument: -%c = %s\n", current_argument, argv[i]); + default: + //throw std::invalid_argument("Invalid argument: " + std::string(argv[i])); + fuse_options.push_back(argv[i]); // could be values that correspond to fuse arguments, such as: -o, -f, etc. break; } - } - else if(argv[i][0] == '-') { + + current_argument = '\0';// reset + } else if(argv[i][0] == '-') { if(strlen(argv[i]) == 1) { - // error + throw std::invalid_argument("Invalid argument: " + std::string(argv[i])); } - else if(argv[i][1] == '2') { // implement flags + + switch(argv[i][1]) { + case '2': // a fastafs specific flag fi->from_fastafs = false; - } - else { // remaining are no flags but arguments with values that need to be parsed + break; + + case 'f': // fuse specific flags + case 's': + case 'd': + fuse_options.push_back(argv[i]); + break; + + case 'o': // argument, fuse specific current_argument = argv[i][1]; - printf(" -> parsing arg: %c\n", current_argument); - - if(argv[i][1] == 'o' or argv[i][1] == 'd' or argv[i][1] == 'f') { - //argv_fuse[fi->argc_fuse++] = argv[i]; - fuse_options.push_back(argv[i]); - } + fuse_options.push_back(argv[i]); // fuse specific + break; + + default: // argument, fastafs spcific (such as '-p' followed by '50') + current_argument = argv[i][1]; + break; } + } else { + full_args.push_back(i); // args such as 'sudo', 'fastafs' 'fastafs-uid' and '/mount/point' of which only the last 2 are needed } - else { - full_args.push_back(i); - printf(" candidate of std-args of which last 2 are needed\n"); - } - i++; - } if(full_args.size() >= 2) { @@ -479,31 +481,23 @@ fuse_instance *parse_args(int argc, char **argv, char **argv_fuse) name = std::filesystem::path(fname).filename(); // remove .fastafs suffix - size_t lastindex = name.find_last_of("."); - name = name.substr(0, lastindex); + size_t lastindex = name.find_last_of("."); + name = name.substr(0, lastindex); - } - else { + } else { name = std::string(argv[mount_target_arg]); } - printf("[%s : %s]\n", name.c_str(), fname.c_str()); - fi->f = new fastafs(name); - printf(" new \n"); - fi->f->load(fname); - printf("loaded!\n"); - fi->cache = fi->f->init_ffs2f(fi->padding, true);// allow mixed case - printf("initiated cache!\n"); } else { std::string basename = std::filesystem::path(std::string(argv[mount_target_arg])).filename(); fi->u2b = new ucsc2bit(basename);// useses basename as prefix for filenames to mount: hg19.2bit -> hg19.2bit.fa fi->u2b->load(std::string(argv[mount_target_arg])); } - + //argv_fuse[fi->argc_fuse++] = argv[mount_target_arg]; argv_fuse[fi->argc_fuse++] = argv[mount_target_arg + 1]; for(size_t j = 0; j < fuse_options.size(); j++) { diff --git a/src/main.cpp b/src/main.cpp index af85b5f1..b58bb48a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -77,7 +77,12 @@ int main(int argc, char *argv[]) if(strcmp(argv[1], "--help") == 0 or strcmp(argv[1], "-h") == 0) { usage(); } else if(strcmp(argv[1], "--version") == 0) { - std::cout << PACKAGE << " v" << PACKAGE_VERSION << GIT_SHA1_STRING << "\n\n"; +#if DEBUG + std::cout << PACKAGE << " v" << PACKAGE_VERSION << GIT_SHA1_STRING << "-debug\n\n"; +#else + std::cout << PACKAGE << " v" << PACKAGE_VERSION << GIT_SHA1_STRING << "-release\n\n"; +#endif //DEBUG + std::cout << "Copyright (C) 2017 Youri Hoogstrate." << "\n"; std::cout << "License GPLv2+: GNU GPL version 2 or later .\n"; std::cout << "This is free software: you are free to change and redistribute it.\n";