From 6d005d381870d99dade4fd68ca022b2eec7783bf Mon Sep 17 00:00:00 2001 From: Mo Morsi Date: Tue, 25 Jun 2019 17:21:32 -0400 Subject: [PATCH 1/2] Add dir based open & create calls Init files from default names under dir --- .../nudb/_experimental/test/test_store.hpp | 23 ++++ include/nudb/basic_store.hpp | 34 +++++ include/nudb/create.hpp | 25 ++++ include/nudb/detail/format.hpp | 18 +++ include/nudb/error.hpp | 5 +- include/nudb/file.hpp | 18 +++ include/nudb/impl/basic_store.ipp | 26 ++++ include/nudb/impl/create.ipp | 40 ++++++ include/nudb/impl/file.ipp | 128 ++++++++++++++++++ test/create.cpp | 19 +++ 10 files changed, 335 insertions(+), 1 deletion(-) create mode 100644 include/nudb/impl/file.ipp diff --git a/include/nudb/_experimental/test/test_store.hpp b/include/nudb/_experimental/test/test_store.hpp index bafe00b..c66bb4d 100644 --- a/include/nudb/_experimental/test/test_store.hpp +++ b/include/nudb/_experimental/test/test_store.hpp @@ -201,10 +201,12 @@ class basic_test_store temp_dir td_; std::uniform_int_distribution sizef_; std::function createf_; + std::function create_dirf_; std::function openf_; Buffer buf_; public: + path_type const dirp; path_type const dp; path_type const kp; path_type const lp; @@ -234,6 +236,9 @@ class basic_test_store void create(error_code& ec); + void + create_dir(error_code& ec); + void open(error_code& ec); @@ -270,11 +275,20 @@ basic_test_store::basic_test_store( keySize, blockSize, loadFactor, ec, args...); }) + , create_dirf_( + [this, args...](error_code& ec) + { + nudb::create( + dirp, appnum, salt, + keySize, blockSize, loadFactor, ec, + args...); + }) , openf_( [this, args...](error_code& ec) { db.open(dp, kp, lp, ec, args...); }) + , dirp(td_.path()) , dp(td_.file("nudb.dat")) , kp(td_.file("nudb.key")) , lp(td_.file("nudb.log")) @@ -330,6 +344,15 @@ create(error_code& ec) createf_(ec); } +template +void +basic_test_store:: +create_dir(error_code& ec) +{ + create_dirf_(ec); +} + + template void basic_test_store:: diff --git a/include/nudb/basic_store.hpp b/include/nudb/basic_store.hpp index 445e2e6..a19442f 100644 --- a/include/nudb/basic_store.hpp +++ b/include/nudb/basic_store.hpp @@ -454,6 +454,40 @@ class basic_store flush() override; }; +/** Open a database. + + The database identified by the specified directory + under which default data and key paths are opened. + + @par Requirements + + The database must be not be open. + + @par Thread safety + + Not thread safe. The caller is responsible for + ensuring that no other store member functions are + called concurrently. + + @param dir_path The path to the directory containing data + and key files. + + @param store The store to open. + + @param ec Set to the error, if any occurred. + + @param args Optional arguments passed to @b File constructors. + +*/ +template +void +open_dir( + path_type const& dir_path, + basic_store& store, + error_code& ec, + Args&&... args); + + } // nudb #include diff --git a/include/nudb/create.hpp b/include/nudb/create.hpp index cb37ecb..13a5cc2 100644 --- a/include/nudb/create.hpp +++ b/include/nudb/create.hpp @@ -110,6 +110,31 @@ create( error_code& ec, Args&&... args); +/** Create a new database in specified directory. + + Similar to create method with explicit dat, + key, and log paths, with the default filenames + being used. + + @param dir_path The path to the data file. + +*/ +template< + class Hasher, + class File = native_file, + class... Args +> +void +create( + path_type const& dir_path, + std::uint64_t appnum, + std::uint64_t salt, + nsize_t key_size, + nsize_t blockSize, + float load_factor, + error_code& ec, + Args&&... args); + } // nudb #include diff --git a/include/nudb/detail/format.hpp b/include/nudb/detail/format.hpp index 1711f51..cbef47d 100644 --- a/include/nudb/detail/format.hpp +++ b/include/nudb/detail/format.hpp @@ -46,6 +46,24 @@ value size up to 32 bits (or 32-bit builds can't read it) static std::size_t constexpr currentVersion = 2; +inline +const path_type& default_dat_file() { + static const path_type ddf("nudb.dat"); + return ddf; +} + +inline +const path_type& default_key_file() { + static const path_type dkf("nudb.key"); + return dkf; +} + +inline +const path_type& default_log_file() { + static const path_type dkf("nudb.log"); + return dkf; +} + struct dat_file_header { static std::size_t constexpr size = diff --git a/include/nudb/error.hpp b/include/nudb/error.hpp index f5f8685..f91553a 100644 --- a/include/nudb/error.hpp +++ b/include/nudb/error.hpp @@ -221,7 +221,10 @@ enum class error size_mismatch, /// duplicate value - duplicate_value + duplicate_value, + + /// directory not found + dir_not_found }; /// Returns the error category used for database error codes. diff --git a/include/nudb/file.hpp b/include/nudb/file.hpp index ba2d367..2dd81ea 100644 --- a/include/nudb/file.hpp +++ b/include/nudb/file.hpp @@ -50,6 +50,24 @@ enum class file_mode write }; +// Return boolean indicating if path exists +bool path_exists(path_type const& path); + +// Return boolean indicating if path is a directory +bool is_dir(path_type const& path); + +// Recursively make the specified dir tree +bool mkdir_p(path_type const& path); + +// Append an rel-path to a local filesystem path. +// The returned path is normalized for the platform. +path_type +path_cat( + path_type const& base, + path_type const& path); + } // nudb +#include + #endif diff --git a/include/nudb/impl/basic_store.ipp b/include/nudb/impl/basic_store.ipp index 4ac79e4..981ffa6 100644 --- a/include/nudb/impl/basic_store.ipp +++ b/include/nudb/impl/basic_store.ipp @@ -784,6 +784,32 @@ flush() ecb_.store(true); } +template +void +open_dir( + path_type const& dir_path, + basic_store& store, + error_code& ec, + Args&&... args) +{ + if(is_dir(dir_path)){ + ec = error::dir_not_found; + return; + } + + path_type dat_path = path_cat(dir_path, + detail::default_dat_file()); + + path_type key_path = path_cat(dir_path, + detail::default_key_file()); + + path_type log_path = path_cat(dir_path, + detail::default_log_file()); + + store.open(dat_path, key_path, log_path, + ec, args...); +} + } // nudb #endif diff --git a/include/nudb/impl/create.ipp b/include/nudb/impl/create.ipp index b0b8511..6b71b9a 100644 --- a/include/nudb/impl/create.ipp +++ b/include/nudb/impl/create.ipp @@ -158,6 +158,46 @@ fail: erase_file(log_path); } +template< + class Hasher, + class File, + class... Args +> +void +create( + path_type const& dir_path, + std::uint64_t appnum, + std::uint64_t salt, + nsize_t key_size, + nsize_t blockSize, + float load_factor, + error_code& ec, + Args&&... args) +{ + if(!is_dir(dir_path)) + mkdir_p(dir_path); + + path_type dat_path = path_cat(dir_path, + detail::default_dat_file()); + + path_type key_path = path_cat(dir_path, + detail::default_key_file()); + + path_type log_path = path_cat(dir_path, + detail::default_log_file()); + + create(dat_path, + key_path, + log_path, + appnum, + salt, + key_size, + blockSize, + load_factor, + ec, + args...); +} + } // nudb #endif diff --git a/include/nudb/impl/file.ipp b/include/nudb/impl/file.ipp new file mode 100644 index 0000000..b09b73c --- /dev/null +++ b/include/nudb/impl/file.ipp @@ -0,0 +1,128 @@ +// +// Copyright (c) 2015-2016 Vinnie Falco (vinnie dot falco at gmail dot com) +// (c) 2019 Mo Morsi (mo at devnull dot network) +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// + +#ifndef NUDB_IMPL_FILE_IPP +#define NUDB_IMPL_FILE_IPP + +#ifdef _MSC_VER +#include +#include // _mkdir +#else +#include +#include +#endif + +namespace nudb { + +inline +bool path_exists(path_type const& path){ +#ifdef _MSC_VER + return GetFileAttributesA(path.c_str()) != + INVALID_FILE_ATTRIBUTES; + +#else + struct stat buf; + return (stat(path.c_str(), &buf) == 0); +#endif +} + +inline +bool is_dir(path_type const& path){ +#ifdef _MSC_VER + DWORD attrs = GetFileAttributesA(path.c_str()); + if (attrs == INVALID_FILE_ATTRIBUTES) + return false; //something is wrong with your path! + + if (attrs & FILE_ATTRIBUTE_DIRECTORY) + return true; // this is a directory! + + return false; + +#else + struct stat buf; + if(stat(path.c_str(), &buf) != 0) + return false; + + return buf.st_mode & S_IFDIR; +#endif +} + +inline +path_type +path_cat( + path_type const& base, + path_type const& path) +{ + if(base.empty()) + return std::string(path); + std::string result(base); +#ifdef BOOST_MSVC + char constexpr path_separator = '\\'; + if(result.back() == path_separator) + result.resize(result.size() - 1); + result.append(path.data(), path.size()); + for(auto& c : result) + if(c == '/') + c = path_separator; +#else + char constexpr path_separator = '/'; + if(result.back() == path_separator) + result.resize(result.size() - 1); + result.append("/"); + result.append(path.data(), path.size()); +#endif + return result; +} + +inline +bool mkdir_p(path_type const& path) +{ +#if _MSC_VER + int ret = _mkdir(path.c_str()); +#else + mode_t mode = 0755; + int ret = mkdir(path.c_str(), mode); +#endif + if (ret == 0) + return true; + + switch (errno) + { + case ENOENT: + // create parent + { + int pos = path.find_last_of('/'); + if ((size_t)pos == std::string::npos) +#if _MSC_VER + pos = path.find_last_of('\\'); + if ((size_t)pos == std::string::npos) +#endif + return false; + if (!mkdir_p( path.substr(0, pos) )) + return false; + } + + // create again +#if _MSC_VER + return 0 == _mkdir(path.c_str()); +#else + return 0 == mkdir(path.c_str(), mode); +#endif + + case EEXIST: + // done! + return is_dir(path); + + default: + return false; + } +} + +} // nudb + +#endif diff --git a/test/create.cpp b/test/create.cpp index 82760d9..9e2c659 100644 --- a/test/create.cpp +++ b/test/create.cpp @@ -38,10 +38,29 @@ class create_test : public boost::beast::unit_test::suite return; } + void + test_create_dir() + { + std::size_t const keySize = 8; + std::size_t const blockSize = 256; + float const loadFactor = 0.5f; + + error_code ec; + test_store ts{keySize, blockSize, loadFactor}; + ts.create_dir(ec); + if(! BEAST_EXPECTS(! ec, ec.message())) + return; + ts.create_dir(ec); + if(! BEAST_EXPECTS( + ec == errc::file_exists, ec.message())) + return; + } + void run() override { test_create(); + test_create_dir(); } }; From 69b325e32e97e76b2daa070aaccb8cce6c2d3da7 Mon Sep 17 00:00:00 2001 From: Mo Morsi Date: Tue, 25 Jun 2019 17:17:23 -0400 Subject: [PATCH 2/2] Add Read only Mode, does not allow inserts --- examples/example.cpp | 2 +- .../nudb/_experimental/test/test_store.hpp | 2 +- include/nudb/basic_store.hpp | 29 ++++++++++ include/nudb/impl/basic_store.ipp | 57 +++++++++++++++---- test/recover.cpp | 4 +- 5 files changed, 80 insertions(+), 14 deletions(-) diff --git a/examples/example.cpp b/examples/example.cpp index 88c4b12..63c124a 100644 --- a/examples/example.cpp +++ b/examples/example.cpp @@ -27,7 +27,7 @@ int main() 0.5f, ec); store db; - db.open(dat_path, key_path, log_path, ec); + db.open(dat_path, key_path, log_path, store::open_mode::read_write, ec); char data = 0; // Insert for(key_type i = 0; i < N; ++i) diff --git a/include/nudb/_experimental/test/test_store.hpp b/include/nudb/_experimental/test/test_store.hpp index c66bb4d..1621334 100644 --- a/include/nudb/_experimental/test/test_store.hpp +++ b/include/nudb/_experimental/test/test_store.hpp @@ -286,7 +286,7 @@ basic_test_store::basic_test_store( , openf_( [this, args...](error_code& ec) { - db.open(dp, kp, lp, ec, args...); + db.open(dp, kp, lp, basic_store::open_mode::read_write, ec, args...); }) , dirp(td_.path()) , dp(td_.file("nudb.dat")) diff --git a/include/nudb/basic_store.hpp b/include/nudb/basic_store.hpp index a19442f..54cd870 100644 --- a/include/nudb/basic_store.hpp +++ b/include/nudb/basic_store.hpp @@ -52,6 +52,14 @@ class basic_store using hash_type = Hasher; using file_type = File; + enum class open_mode { + /// Open the database for both read and write access + read_write, + + /// Open the database for reach access only + read + }; + private: using clock_type = std::chrono::steady_clock; @@ -90,9 +98,14 @@ class basic_store path_type const& dp_, path_type const& kp_, path_type const& lp_, detail::key_file_header const& kh_); + + state(File&& df_, File&& kf_, + path_type const& dp_, path_type const& kp_, + detail::key_file_header const& kh_); }; bool open_ = false; + bool is_writable_; // Use optional because some // members cannot be default-constructed. @@ -173,6 +186,19 @@ class basic_store return open_; } + /** Returns `true` if the database is writable + + @par Thread safety + + Safe to call concurrently with any function + except @ref open. + */ + bool + is_writable() const + { + return is_writable_; + } + /** Return the path to the data file. @par Requirements @@ -329,6 +355,8 @@ class basic_store @param log_path The path to the log file. + @param open_mode Mode of database access + @param ec Set to the error, if any occurred. @param args Optional arguments passed to @b File constructors. @@ -340,6 +368,7 @@ class basic_store path_type const& dat_path, path_type const& key_path, path_type const& log_path, + open_mode mode, error_code& ec, Args&&... args); diff --git a/include/nudb/impl/basic_store.ipp b/include/nudb/impl/basic_store.ipp index 981ffa6..a2175e7 100644 --- a/include/nudb/impl/basic_store.ipp +++ b/include/nudb/impl/basic_store.ipp @@ -46,6 +46,25 @@ state(File&& df_, File&& kf_, File&& lf_, "File requirements not met"); } +template +basic_store::state:: +state(File&& df_, File&& kf_, + path_type const& dp_, path_type const& kp_, + detail::key_file_header const& kh_) + : df(std::move(df_)) + , kf(std::move(kf_)) + , dp(dp_) + , kp(kp_) + , hasher(kh_.salt) + , p0(kh_.key_size, "p0") + , p1(kh_.key_size, "p1") + , c1(kh_.key_size, kh_.block_size, "c1") + , kh(kh_) +{ + BOOST_STATIC_ASSERT_MSG(is_File::value, + "File requirements not met"); +} + //------------------------------------------------------------------------------ template @@ -122,9 +141,12 @@ open( path_type const& dat_path, path_type const& key_path, path_type const& log_path, + open_mode mode, error_code& ec, Args&&... args) { + bool writable = mode == open_mode::read_write; + static_assert(is_Hasher::value, "Hasher requirements not met"); using namespace detail; @@ -138,17 +160,23 @@ open( File df(args...); File kf(args...); File lf(args...); - df.open(file_mode::append, dat_path, ec); - if(ec) - return; - kf.open(file_mode::write, key_path, ec); + df.open(writable ? file_mode::append : file_mode::read, + dat_path, ec); if(ec) return; - lf.create(file_mode::append, log_path, ec); + kf.open(writable ? file_mode::write : file_mode::read, + key_path, ec); if(ec) return; - // VFALCO TODO Erase empty log file if this - // function subsequently fails. + + if(writable){ + lf.create(file_mode::append, log_path, ec); + if(ec) + return; + // VFALCO TODO Erase empty log file if this + // function subsequently fails. + } + dat_file_header dh; read(df, dh, ec); if(ec) @@ -167,8 +195,12 @@ open( if(ec) return; boost::optional s; - s.emplace(std::move(df), std::move(kf), std::move(lf), - dat_path, key_path, log_path, kh); + if(writable) + s.emplace(std::move(df), std::move(kf), std::move(lf), + dat_path, key_path, log_path, kh); + else + s.emplace(std::move(df), std::move(kf), + dat_path, key_path, kh); thresh_ = std::max(65536UL, kh.load_factor * kh.capacity); frac_ = thresh_ / 2; @@ -180,10 +212,13 @@ open( ec = error::short_key_file; return; } - dataWriteSize_ = 32 * nudb::block_size(dat_path); - logWriteSize_ = 32 * nudb::block_size(log_path); + if(writable){ + dataWriteSize_ = 32 * nudb::block_size(dat_path); + logWriteSize_ = 32 * nudb::block_size(log_path); + } s_.emplace(std::move(*s)); open_ = true; + is_writable_ = writable; ctx_->insert(*this); } diff --git a/test/recover.cpp b/test/recover.cpp index 36b0564..15c642c 100644 --- a/test/recover.cpp +++ b/test/recover.cpp @@ -60,7 +60,9 @@ class basic_recover_test : public boost::beast::unit_test::suite if(ec) return; basic_store> db; - db.open(ts.dp, ts.kp, ts.lp, ec, c); + db.open(ts.dp, ts.kp, ts.lp, + basic_store>::open_mode::read_write, + ec, c); if(ec) return; if(! BEAST_EXPECT(db.appnum() == ts.appnum))