diff --git a/CMakeLists.txt b/CMakeLists.txt index 03d8592f5..88d4bdcf4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -70,6 +70,45 @@ ecbuild_add_option( FEATURE FDB_REMOTE DEFAULT ON DESCRIPTION "Support for FDB remote access" ) +find_package(UUID QUIET) + +find_package(DAOS QUIET) + +set(_default_dummy_daos ON) +if(DAOS_FOUND) + set(_default_dummy_daos OFF) +endif() + +ecbuild_add_option( FEATURE DUMMY_DAOS + DEFAULT ${_default_dummy_daos} + CONDITION UUID_FOUND + DESCRIPTION "Use dummy DAOS library emulating DAOS with a file system" ) + +if(HAVE_DUMMY_DAOS) + set(DAOS_LIBRARIES dummy_daos) + set(DAOS_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/src/dummy_daos) + set(DAOS_FOUND TRUE) + set(DAOS_TESTS_LIBRARIES dummy_daos_tests) + set(DAOS_TESTS_INCLUDE_DIRS ${PROJECT_SOURCE_DIR}/src/dummy_daos) + set(DAOS_TESTS_FOUND TRUE) +endif() + +ecbuild_add_option( FEATURE DAOSFDB + CONDITION DAOS_FOUND AND UUID_FOUND + DEFAULT ON + DESCRIPTION "DAOS support for FDB Store" ) + +ecbuild_add_option( FEATURE DAOS_ADMIN + DEFAULT OFF + CONDITION HAVE_DAOSFDB AND DAOS_TESTS_FOUND + DESCRIPTION "Add features for DAOS pool management. Removes need to manually create a pool for DAOS unit tests" ) + +# DAOS_TESTS is a daos admin library, usually not present +if(NOT HAVE_DAOS_ADMIN) + set( DAOS_TESTS_LIBRARIES "" ) + set( DAOS_TESTS_INCLUDE_DIRS "" ) +endif() + ecbuild_add_option( FEATURE EXPERIMENTAL DEFAULT OFF DESCRIPTION "Experimental features" ) diff --git a/cmake/FindDAOS.cmake b/cmake/FindDAOS.cmake new file mode 100644 index 000000000..23a2224a8 --- /dev/null +++ b/cmake/FindDAOS.cmake @@ -0,0 +1,164 @@ +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +include(FindPackageHandleStandardArgs) + +# daos + +find_path(DAOS_INCLUDE_DIR + NAMES daos.h + HINTS + ${DAOS_ROOT} + ${DAOS_DIR} + ${DAOS_PATH} + ENV DAOS_ROOT + ENV DAOS_DIR + ENV DAOS_PATH + PATH_SUFFIXES include +) + +find_library(DAOS_LIBRARY + NAMES daos + HINTS + ${DAOS_ROOT} + ${DAOS_DIR} + ${DAOS_PATH} + ENV DAOS_ROOT + ENV DAOS_DIR + ENV DAOS_PATH + PATH_SUFFIXES lib lib64 +) + +# daos_common + +find_library(DAOS_COMMON_LIBRARY + NAMES daos_common + HINTS + ${DAOS_ROOT} + ${DAOS_DIR} + ${DAOS_PATH} + ENV DAOS_ROOT + ENV DAOS_DIR + ENV DAOS_PATH + PATH_SUFFIXES lib lib64 +) + +# gurt + +find_library(GURT_LIBRARY + NAMES gurt + HINTS + ${DAOS_ROOT} + ${DAOS_DIR} + ${DAOS_PATH} + ENV DAOS_ROOT + ENV DAOS_DIR + ENV DAOS_PATH + PATH_SUFFIXES lib lib64 +) + +# daos tests + +# if not using dummy DAOS, then the daos_tests lib should be installed +# from daos RPMs into the corresponding system directories. However its +# headers (daos/tests_lib.h) are not provided in any of the RPMs and +# need to be copied from source. tests_lib.h could be copied into standard +# system directories together with other DAOS headers, but the tests_lib.h +# file includes many other DAOS headers not provided by RPMs and other +# external library headers, e.g. boost, and are required when compiling a +# program which includes tests_lib.h (e.g. fdb5's DAOS backend with +# DAOS_ADMIN enabled for unit testing). To avoid having to install all +# these headers, tests_lib.h can be copied in a user directory and modified +# to only declare the necessary functions (pool create and destroy) and +# remove most of the includes. All this explains why a DAOS_TESTS_INCLUDE_ROOT +# environment variable is used here to find tests_lib.h rather than DAOS_ROOT. + +find_path(DAOS_TESTS_INCLUDE_DIR + NAMES daos/tests_lib.h + HINTS + ${DAOS_TESTS_INCLUDE_ROOT} + ${DAOS_TESTS_DIR} + ${DAOS_TESTS_PATH} + ENV DAOS_TESTS_INCLUDE_ROOT + ENV DAOS_TESTS_DIR + ENV DAOS_TESTS_PATH + PATH_SUFFIXES include +) + +find_library(DAOS_TESTS_LIBRARY + NAMES daos_tests + HINTS + ${DAOS_ROOT} + ${DAOS_DIR} + ${DAOS_PATH} + ENV DAOS_ROOT + ENV DAOS_DIR + ENV DAOS_PATH + PATH_SUFFIXES lib lib64 +) + +find_package_handle_standard_args( + DAOS + DEFAULT_MSG + DAOS_LIBRARY + DAOS_INCLUDE_DIR + DAOS_COMMON_LIBRARY + GURT_LIBRARY ) + +mark_as_advanced(DAOS_INCLUDE_DIR DAOS_LIBRARY DAOS_COMMON_LIBRARY GURT_LIBRARY) + +if(DAOS_FOUND) + add_library(daos UNKNOWN IMPORTED GLOBAL) + set_target_properties(daos PROPERTIES + IMPORTED_LOCATION ${DAOS_LIBRARY} + INTERFACE_INCLUDE_DIRECTORIES ${DAOS_INCLUDE_DIR} + ) + add_library(daos_common UNKNOWN IMPORTED GLOBAL) + set_target_properties(daos_common PROPERTIES + IMPORTED_LOCATION ${DAOS_COMMON_LIBRARY} + INTERFACE_INCLUDE_DIRECTORIES ${DAOS_INCLUDE_DIR} + ) + add_library(gurt UNKNOWN IMPORTED GLOBAL) + set_target_properties(gurt PROPERTIES + IMPORTED_LOCATION ${GURT_LIBRARY} + INTERFACE_INCLUDE_DIRECTORIES ${DAOS_INCLUDE_DIR} + ) + set(DAOS_INCLUDE_DIRS ${DAOS_INCLUDE_DIR}) + list(APPEND DAOS_LIBRARIES daos daos_common gurt) +endif() + +find_package_handle_standard_args( + DAOS_TESTS + DEFAULT_MSG + DAOS_TESTS_LIBRARY + DAOS_TESTS_INCLUDE_DIR ) + +mark_as_advanced(DAOS_TESTS_INCLUDE_DIR DAOS_TESTS_LIBRARY) + +if(DAOS_TESTS_FOUND) + add_library(daos_tests UNKNOWN IMPORTED GLOBAL) + set_target_properties(daos_tests PROPERTIES + IMPORTED_LOCATION ${DAOS_TESTS_LIBRARY} + INTERFACE_INCLUDE_DIRECTORIES "${DAOS_TESTS_INCLUDE_DIR};${DAOS_INCLUDE_DIR}" + ) + list(APPEND DAOS_TESTS_INCLUDE_DIRS + ${DAOS_TESTS_INCLUDE_DIR} + ${DAOS_INCLUDE_DIR} + ) + set(DAOS_TESTS_LIBRARIES daos_tests) +endif() diff --git a/cmake/FindUUID.cmake b/cmake/FindUUID.cmake new file mode 100644 index 000000000..0072ed0b0 --- /dev/null +++ b/cmake/FindUUID.cmake @@ -0,0 +1,60 @@ +# Copyright 2022 European Centre for Medium-Range Weather Forecasts (ECMWF) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation nor +# does it submit to any jurisdiction. + +include(FindPackageHandleStandardArgs) + +# uuid + +find_path(UUID_INCLUDE_DIR + NAMES uuid/uuid.h + HINTS + ${UUID_ROOT} + ${UUID_DIR} + ${UUID_PATH} + ENV UUID_ROOT + ENV UUID_DIR + ENV UUID_PATH + PATH_SUFFIXES include include/uuid + NO_DEFAULT_PATH +) + +find_library(UUID_LIBRARY + NAMES uuid + HINTS + ${UUID_ROOT} + ${UUID_DIR} + ${UUID_PATH} + ENV UUID_ROOT + ENV UUID_DIR + ENV UUID_PATH + PATH_SUFFIXES lib lib64 +) + +find_package_handle_standard_args(UUID DEFAULT_MSG UUID_LIBRARY UUID_INCLUDE_DIR) + +mark_as_advanced(UUID_INCLUDE_DIR UUID_LIBRARY) + +if(UUID_FOUND) + add_library(uuid UNKNOWN IMPORTED GLOBAL) + set_target_properties(uuid PROPERTIES + IMPORTED_LOCATION ${UUID_LIBRARY} + INTERFACE_INCLUDE_DIRECTORIES ${UUID_INCLUDE_DIR} + ) + set(UUID_INCLUDE_DIRS ${UUID_INCLUDE_DIR}) + set(UUID_LIBRARIES uuid) +endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 93989f762..a4dac3667 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,6 +5,10 @@ ecbuild_find_project_files() add_subdirectory( fdb5 ) +if (HAVE_DUMMY_DAOS) + add_subdirectory(dummy_daos) +endif() + ############################################################################################ # following directories are not packaged # currently this is developer only, hence not distributed diff --git a/src/dummy_daos/CMakeLists.txt b/src/dummy_daos/CMakeLists.txt new file mode 100644 index 000000000..7339d1d3e --- /dev/null +++ b/src/dummy_daos/CMakeLists.txt @@ -0,0 +1,45 @@ +ecbuild_add_library( + + TARGET dummy_daos + + INSTALL_HEADERS_LIST + daos.h + + HEADER_DESTINATION ${INSTALL_INCLUDE_DIR} + + SOURCES + daos.h + daos.cc + dummy_daos.h + dummy_daos.cc + + PUBLIC_LIBS + eckit + uuid + +) + +if(HAVE_DAOS_ADMIN) + + ecbuild_add_library( + + TARGET dummy_daos_tests + + INSTALL_HEADERS LISTED + + HEADER_DESTINATION ${INSTALL_INCLUDE_DIR} + + SOURCES + daos/tests_lib.h + daos/tests_lib.cc + + PRIVATE_INCLUDES + ${PROJECT_SOURCE_DIR}/../daos.h + ${PROJECT_SOURCE_DIR}/../dummy_daos.h + + PRIVATE_LIBS + dummy_daos + + ) + +endif() diff --git a/src/dummy_daos/daos.cc b/src/dummy_daos/daos.cc new file mode 100644 index 000000000..11fde5816 --- /dev/null +++ b/src/dummy_daos/daos.cc @@ -0,0 +1,1154 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * @file daos.cc + * @author Nicolau Manubens + * @date Jun 2022 + */ + +#include +#include +#include +#include +#include + +#include "eckit/runtime/Main.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/io/Length.h" +#include "eckit/io/FileHandle.h" +#include "eckit/types/UUID.h" +#include "eckit/log/TimeStamp.h" +#include "eckit/utils/MD5.h" + +#include "daos.h" +#include "dummy_daos.h" + +using eckit::PathName; + +namespace { + void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); + }; +} + +extern "C" { + +//---------------------------------------------------------------------------------------------------------------------- + +typedef struct daos_handle_internal_t { + PathName path; +} daos_handle_internal_t; + +//---------------------------------------------------------------------------------------------------------------------- + +int daos_init() { + const char* argv[2] = {"dummy-daos-api", 0}; + eckit::Main::initialise(1, const_cast(argv)); + PathName root = dummy_daos_root(); + root.mkdir(); + return 0; +} + +int daos_fini() { + return 0; +} + +int daos_pool_connect(const char *pool, const char *sys, unsigned int flags, + daos_handle_t *poh, daos_pool_info_t *info, daos_event_t *ev) { + + poh->impl = nullptr; + + if (sys != NULL) NOTIMP; + if (flags != DAOS_PC_RW) NOTIMP; + if (info != NULL) NOTIMP; + if (ev != NULL) NOTIMP; + + eckit::PathName path = dummy_daos_root() / pool; + eckit::PathName realpath{dummy_daos_root()}; + + uuid_t uuid = {0}; + if (uuid_parse(pool, uuid) == 0) { + + realpath /= pool; + + } else { + + try { + + ASSERT(path.isLink()); + realpath /= path.realName().baseName(); + + } catch (eckit::FailedSystemCall& e) { + + if (path.exists()) throw; + return -1; + + } + + } + + if (!realpath.exists()) return -1; + + std::unique_ptr impl(new daos_handle_internal_t); + impl->path = realpath; + poh->impl = impl.release(); + + return 0; + +} + +int daos_pool_disconnect(daos_handle_t poh, daos_event_t *ev) { + + ASSERT(poh.impl); + delete poh.impl; + + if (ev != NULL) NOTIMP; + + return 0; + +} + +int daos_pool_list_cont(daos_handle_t poh, daos_size_t *ncont, + struct daos_pool_cont_info *cbuf, daos_event_t *ev) { + + ASSERT(poh.impl); + if (ev != NULL) NOTIMP; + + daos_size_t n(*ncont); + + std::vector files; + std::vector dirs; + + poh.impl->path.children(files, dirs); + + *ncont = files.size(); + + if (cbuf == NULL) return 0; + + if (files.size() > n) return -1; + + daos_size_t nfound = 0; + + for (auto& f : files) { + + if (f.exists()) { /// @todo: is the check in this line really necessary, given the try-catch below? + + ++nfound; + + try { + + ASSERT(f.isLink()); + std::string contname = f.baseName(); + std::string uuid_str = f.realName().baseName(); + + if (contname.rfind("__dummy_daos_uuid_", 0) != 0) { + const char* contname_cstr = contname.c_str(); + ASSERT(strlen(contname_cstr) <= DAOS_PROP_LABEL_MAX_LEN); + strncpy(cbuf[nfound - 1].pci_label, contname_cstr, DAOS_PROP_LABEL_MAX_LEN + 1); + } + + const char* uuid_cstr = uuid_str.c_str(); + uuid_t uuid = {0}; + ASSERT(uuid_parse(uuid_cstr, uuid) == 0); + uuid_copy(cbuf[nfound - 1].pci_uuid, uuid); + + } catch (eckit::FailedSystemCall& e) { + + if (f.exists()) throw; + --nfound; + + } + } + + } + + *ncont = nfound; + + return 0; + +} + +int daos_cont_create_internal(daos_handle_t poh, uuid_t *uuid) { + + ASSERT(poh.impl); + + /// @note: name generation copied from LocalPathName::unique. Ditched StaticMutex + /// as dummy DAOS is not thread safe. + + std::string hostname = eckit::Main::hostname(); + + static unsigned long long n = (((unsigned long long)::getpid()) << 32); + + static std::string format = "%Y%m%d.%H%M%S"; + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + + std::string name = os.str(); + + while (::access(name.c_str(), F_OK) == 0) { + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + name = os.str(); + } + + uuid_t new_uuid = {0}; + eckit::MD5 md5(name); + uint64_t hi = std::stoull(md5.digest().substr(0, 8), nullptr, 16); + uint64_t lo = std::stoull(md5.digest().substr(8, 16), nullptr, 16); + ::memcpy(&new_uuid[0], &hi, sizeof(hi)); + ::memcpy(&new_uuid[8], &lo, sizeof(lo)); + + char cont_uuid_cstr[37] = ""; + uuid_unparse(new_uuid, cont_uuid_cstr); + + eckit::PathName cont_path = poh.impl->path / cont_uuid_cstr; + + if (cont_path.exists()) throw eckit::SeriousBug("UUID clash in cont create"); + + cont_path.mkdir(); + + if (uuid != NULL) uuid_copy(*uuid, new_uuid); + + return 0; + +} + +/// @note: containers are implemented as directories within pool directories. Upon creation, a +/// container directory is named with a newly generated UUID. If a label is specified, a +/// symlink is created with the label as origin file name and the UUID as target directory. +/// If no label is specified, a similr symlink is created with the UUID with the +/// "__dummy_daos_uuid_" prefix as origin file name and the UUID as target directory. +/// This mechanism is necessary for listing and removing containers under concurrent, +/// potentially racing container operations. + +int daos_cont_create(daos_handle_t poh, uuid_t *uuid, daos_prop_t *cont_prop, daos_event_t *ev) { + + ASSERT(poh.impl); + + if (cont_prop != NULL && cont_prop->dpp_entries) { + + if (cont_prop->dpp_nr != 1) NOTIMP; + if (cont_prop->dpp_entries[0].dpe_type != DAOS_PROP_CO_LABEL) NOTIMP; + + struct daos_prop_entry *entry = &cont_prop->dpp_entries[0]; + + if (entry == NULL) NOTIMP; + + std::string cont_name{entry->dpe_str}; + + return daos_cont_create_with_label(poh, cont_name.c_str(), NULL, uuid, ev); + + } + + if (ev != NULL) NOTIMP; + + uuid_t new_uuid = {0}; + + ASSERT(daos_cont_create_internal(poh, &new_uuid) == 0); + + char cont_uuid_cstr[37] = ""; + uuid_unparse(new_uuid, cont_uuid_cstr); + + eckit::PathName label_symlink_path = poh.impl->path / (std::string("__dummy_daos_uuid_") + cont_uuid_cstr); + + eckit::PathName cont_path = poh.impl->path / cont_uuid_cstr; + + if (::symlink(cont_path.path().c_str(), label_symlink_path.path().c_str()) < 0) { + + if (errno == EEXIST) { // link path already exists due to race condition + + throw eckit::SeriousBug("unexpected race condition in unnamed container symlink creation"); + + } else { // symlink fails for unknown reason + + throw eckit::FailedSystemCall(std::string("symlink ") + cont_path.path() + " " + label_symlink_path.path()); + + } + + } + + if (uuid != NULL) uuid_copy(*uuid, new_uuid); + + return 0; + +} + +int daos_cont_create_with_label(daos_handle_t poh, const char *label, + daos_prop_t *cont_prop, uuid_t *uuid, + daos_event_t *ev) { + + ASSERT(poh.impl); + if (cont_prop != NULL) NOTIMP; + if (ev != NULL) NOTIMP; + + ASSERT(std::string{label}.rfind("__dummy_daos_uuid_", 0) != 0); + ASSERT(strlen(label) <= DAOS_PROP_LABEL_MAX_LEN); + + eckit::PathName label_symlink_path = poh.impl->path / label; + + if (label_symlink_path.exists()) return 0; + + uuid_t new_uuid = {0}; + + ASSERT(daos_cont_create_internal(poh, &new_uuid) == 0); + + char cont_uuid_cstr[37] = ""; + uuid_unparse(new_uuid, cont_uuid_cstr); + + eckit::PathName cont_path = poh.impl->path / cont_uuid_cstr; + + if (::symlink(cont_path.path().c_str(), label_symlink_path.path().c_str()) < 0) { + + if (errno == EEXIST) { // link path already exists due to race condition + + if (uuid != NULL) { + /// @todo: again might find race condition here: + std::string found_uuid = label_symlink_path.realName().baseName(); + uuid_parse(found_uuid.c_str(), *uuid); + } + + deldir(cont_path); + + return 0; + + } else { // symlink fails for unknown reason + + throw eckit::FailedSystemCall(std::string("symlink ") + cont_path.path() + " " + label_symlink_path.path()); + + } + + } + + if (uuid != NULL) uuid_copy(*uuid, new_uuid); + + return 0; + +} + +/// @note: in DAOS, an attempt to destroy a container with open handles results +/// in error by default. This behavior is not implemented in dummy DAOS. +/// In DAOS, an attempt to destroy a container with open handles with the +/// force flag enabled closes open handles, and therefore ongoing/future +/// operations on these handles fail. The contained objects and the +/// container are immediately destroyed. In dummy DAOS the open handles +/// are implemented with file descriptors, and these are left open. A +/// remove operation is called on the corresponding file names but the +/// descriptors remain open. Therefore, in contrast to DAOS, +/// ongoing/future operations on these handles succeed, and the files +/// are destroyed after the descriptors are closed. The folder +/// implementing the container, however, is immediately removed. + +int daos_cont_destroy(daos_handle_t poh, const char *cont, int force, daos_event_t *ev) { + + ASSERT(poh.impl); + if (force != 1) NOTIMP; + if (ev != NULL) NOTIMP; + + ASSERT(std::string{cont}.rfind("__dummy_daos_uuid_", 0) != 0); + + eckit::PathName path = poh.impl->path; + uuid_t uuid = {0}; + if (uuid_parse(cont, uuid) == 0) { + path /= std::string("__dummy_daos_uuid_") + cont; + } else { + path /= cont; + } + + eckit::PathName realpath(""); + try { + + ASSERT(path.isLink()); + realpath = path.realName(); + path.unlink(); + + } catch (eckit::FailedSystemCall& e) { + + if (path.exists()) throw; + return -DER_NONEXIST; + + } + + try { + + deldir(realpath); + + } catch (eckit::FailedSystemCall& e) { + + if (realpath.exists()) throw; + return -DER_NONEXIST; + + } + + return 0; + +} + +int daos_cont_open(daos_handle_t poh, const char *cont, unsigned int flags, daos_handle_t *coh, + daos_cont_info_t *info, daos_event_t *ev) { + + ASSERT(poh.impl); + if (flags != DAOS_COO_RW) NOTIMP; + if (info != NULL) NOTIMP; + if (ev != NULL) NOTIMP; + + ASSERT(std::string{cont}.rfind("__dummy_daos_uuid_", 0) != 0); + + eckit::PathName path = poh.impl->path; + uuid_t uuid = {0}; + if (uuid_parse(cont, uuid) == 0) { + path /= std::string("__dummy_daos_uuid_") + cont; + } else { + path /= cont; + } + + if (!path.exists()) return -DER_NONEXIST; + + eckit::PathName realpath{poh.impl->path}; + try { + + ASSERT(path.isLink()); + realpath /= path.realName().baseName(); + + } catch (eckit::FailedSystemCall& e) { + + if (path.exists()) throw; + return -DER_NONEXIST; + + } + + if (!realpath.exists()) return -DER_NONEXIST; + + std::unique_ptr impl(new daos_handle_internal_t); + impl->path = realpath; + + coh->impl = impl.release(); + return 0; + +} + +int daos_cont_close(daos_handle_t coh, daos_event_t *ev) { + + ASSERT(coh.impl); + delete coh.impl; + + if (ev != NULL) NOTIMP; + + return 0; + +} + +int daos_cont_alloc_oids(daos_handle_t coh, daos_size_t num_oids, uint64_t *oid, + daos_event_t *ev) { + + static uint64_t next_oid = 0; + + ASSERT(coh.impl); + if (ev != NULL) NOTIMP; + ASSERT(num_oids > (uint64_t) 0); + + // support for multi-node clients running dummy DAOS backed by a + // distributed file system + std::string host = eckit::Main::instance().hostname(); + + uuid_t uuid = {0}; + eckit::MD5 md5(host); + uint64_t hi = std::stoull(md5.digest().substr(0, 8), nullptr, 16); + uint64_t lo = std::stoull(md5.digest().substr(8, 16), nullptr, 16); + ::memcpy(&uuid[0], &hi, sizeof(hi)); + ::memcpy(&uuid[8], &lo, sizeof(lo)); + + char uuid_cstr[37] = ""; + uuid_unparse(uuid, uuid_cstr); + + pid_t pid = getpid(); + + // only 20 out of the 32 bits in pid_t are used to identify the calling process. + // because of this, there could be oid clashes + uint64_t pid_mask = 0x00000000000FFFFF; + + uint64_t oid_mask = 0x000000000FFFFFFF; + ASSERT((next_oid + num_oids) <= oid_mask); + + *oid = next_oid; + *oid |= (((uint64_t) pid) & pid_mask) << 28; + *oid |= (((uint64_t) *(((unsigned char *) uuid) + 1)) << 48); + *oid |= (((uint64_t) *((unsigned char *) uuid)) << 56); + + next_oid += num_oids; + + return 0; + +} + +int daos_obj_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, + enum daos_otype_t type, daos_oclass_id_t cid, + daos_oclass_hints_t hints, uint32_t args) { + + ASSERT(coh.impl); + if (type != DAOS_OT_KV_HASHED && type != DAOS_OT_ARRAY && type != DAOS_OT_ARRAY_BYTE) NOTIMP; + if (cid != OC_S1) NOTIMP; + if (hints != 0) NOTIMP; + if (args != 0) NOTIMP; + + oid->hi &= (uint64_t) 0x00000000FFFFFFFF; + oid->hi |= ((((uint64_t) type) & OID_FMT_TYPE_MAX) << OID_FMT_TYPE_SHIFT); + oid->hi |= ((((uint64_t) cid) & OID_FMT_CLASS_MAX) << OID_FMT_CLASS_SHIFT); + + return 0; + +} + +/// @note: real daos_kv_open does not involve RPC whereas dummy daos requires +/// interaction with the file system +int daos_kv_open(daos_handle_t coh, daos_obj_id_t oid, unsigned int mode, + daos_handle_t *oh, daos_event_t *ev) { + + ASSERT(coh.impl); + if (mode != DAOS_OO_RW) NOTIMP; + if (ev != NULL) NOTIMP; + + std::stringstream os; + os << std::setw(16) << std::setfill('0') << std::hex << oid.hi; + os << "."; + os << std::setw(16) << std::setfill('0') << std::hex << oid.lo; + + std::unique_ptr impl(new daos_handle_internal_t); + impl->path = coh.impl->path / os.str(); + + impl->path.mkdir(); + + oh->impl = impl.release(); + return 0; + +} + +/// @note: destruction of KVs with open handles may not be consistent. Notes +/// in daos_cont_destroy apply here too. + +int daos_kv_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev) { + + ASSERT(oh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (ev != NULL) NOTIMP; + + try { + + deldir(oh.impl->path); + + } catch (eckit::FailedSystemCall& e) { + + if (oh.impl->path.exists()) throw; + return -DER_NONEXIST; + + } + + return 0; + +} + +int daos_obj_close(daos_handle_t oh, daos_event_t *ev) { + + ASSERT(oh.impl); + delete oh.impl; + + if (ev != NULL) NOTIMP; + + return 0; + +} + +// daos_kv_put and get are only guaranteed to work if values of a same fixed size are put/get. +// e.g. two racing processes could both openForWrite as part of daos_kv_put (and wipe file +// content), then one writes (puts) content of length 2*x, the other writes content of +// length x, but old content remains at the end from the first kv_put. +// e.g. a process could openForRead as part of daos_kv_get (which retrieves content length), +// then another raching process could daos_kv_put of some content with different length, +// and then the first process would resume retrieval and obtain content of unexpected length. +// if so, daos_kv_put and get are transactional + +int daos_kv_put(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char *key, + daos_size_t size, const void *buf, daos_event_t *ev) { + + ASSERT(oh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (flags != 0) NOTIMP; + if (ev != NULL) NOTIMP; + + eckit::FileHandle fh(oh.impl->path / key, true); + fh.openForWrite(eckit::Length(size)); + eckit::AutoClose closer(fh); + long res = fh.write(buf, (long) size); + ASSERT(res == (long) size); + + return 0; + +} + +int daos_kv_get(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char *key, + daos_size_t *size, void *buf, daos_event_t *ev) { + + ASSERT(oh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (flags != 0) NOTIMP; + if (ev != NULL) NOTIMP; + + bool exists = (oh.impl->path / key).exists(); + + if (!exists && buf != NULL) return -DER_NONEXIST; + + daos_size_t dest_size = *size; + *size = 0; + if (!exists) return 0; + + eckit::FileHandle fh(oh.impl->path / key); + eckit::Length len = fh.size(); + *size = len; + + if (buf == NULL) return 0; + + if (len > dest_size) return -1; + + fh.openForRead(); + eckit::AutoClose closer(fh); + long res = fh.read(buf, len); + ASSERT(eckit::Length(res) == len); + + return 0; + +} + +int daos_kv_remove(daos_handle_t oh, daos_handle_t th, uint64_t flags, + const char *key, daos_event_t *ev) { + + ASSERT(oh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (flags != 0) NOTIMP; + if (ev != NULL) NOTIMP; + + if (!oh.impl->path.exists()) return -1; + + /// @todo: should removal of a non-existing key fail? + /// @todo: if not, can the exist check be avoided and unlink be called directly? + if ((oh.impl->path / key).exists()) (oh.impl->path / key).unlink(); + + return 0; + +} + +int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t *nr, + daos_key_desc_t *kds, d_sg_list_t *sgl, daos_anchor_t *anchor, + daos_event_t *ev) { + + ASSERT(oh.impl); + static std::vector ongoing_req; + static std::string req_hash; + static unsigned long long n = (((unsigned long long)::getpid()) << 32); + + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (ev != NULL) NOTIMP; + + if (nr == NULL) return -1; + if (kds == NULL) return -1; + if (sgl == NULL) return -1; + if (sgl->sg_nr != 1) NOTIMP; + if (sgl->sg_iovs == NULL) return -1; + if (anchor == NULL) return -1; + + if (!oh.impl->path.exists()) return -1; + + if (anchor->da_type == DAOS_ANCHOR_TYPE_EOF) return -1; + if (anchor->da_type == DAOS_ANCHOR_TYPE_HKEY) NOTIMP; + + if (anchor->da_type == DAOS_ANCHOR_TYPE_ZERO) { + + /// client process must consume all key names before starting a new request + if (ongoing_req.size() != 0) NOTIMP; + + std::vector files; + std::vector dirs; + oh.impl->path.children(files, dirs); + + for (auto& f : files) ongoing_req.push_back(f.baseName()); + + anchor->da_type = DAOS_ANCHOR_TYPE_KEY; + + std::string hostname = eckit::Main::hostname(); + static std::string format = "%Y%m%d.%H%M%S"; + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + std::string name = os.str(); + while (::access(name.c_str(), F_OK) == 0) { + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + name = os.str(); + } + uuid_t new_uuid = {0}; + eckit::MD5 md5(name); + req_hash = md5.digest(); + + ::memcpy((char*) &(anchor->da_buf[0]), req_hash.c_str(), req_hash.size()); + anchor->da_shard = (uint16_t) req_hash.size(); + + } else { + + if (anchor->da_type != DAOS_ANCHOR_TYPE_KEY) + throw eckit::SeriousBug("Unexpected anchor type"); + + /// different processes cannot collaborate on consuming a same kv_list + /// request (i.e. cannot share a hash) + if (std::string((char*) &(anchor->da_buf[0]), anchor->da_shard) != req_hash) NOTIMP; + + } + + size_t remain_size = sgl->sg_iovs[0].iov_buf_len; + uint32_t remain_kds = *nr; + size_t sgl_pos = 0; + *nr = 0; + while (remain_kds > 0 && remain_size > 0 && ongoing_req.size() > 0) { + size_t next_size = ongoing_req.back().size(); + if (next_size > remain_size) { + if (*nr == 0) return -1; + remain_size = 0; + continue; + } + ::memcpy((char*) sgl->sg_iovs[0].iov_buf + sgl_pos, ongoing_req.back().c_str(), next_size); + ongoing_req.pop_back(); + kds[*nr].kd_key_len = next_size; + remain_size -= next_size; + remain_kds--; + sgl_pos += next_size; + *nr += 1; + } + + if (ongoing_req.size() == 0) anchor->da_type = DAOS_ANCHOR_TYPE_EOF; + + return 0; + +} + +int daos_array_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, bool add_attr, daos_oclass_id_t cid, + daos_oclass_hints_t hints, uint32_t args) { + + if (add_attr) { + + return daos_obj_generate_oid(coh, oid, DAOS_OT_ARRAY, cid, hints, args); + + } else { + + return daos_obj_generate_oid(coh, oid, DAOS_OT_ARRAY_BYTE, cid, hints, args); + + } + +} + +int daos_array_create(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, + daos_size_t cell_size, daos_size_t chunk_size, + daos_handle_t *oh, daos_event_t *ev) { + + ASSERT(coh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (ev != NULL) NOTIMP; + + std::stringstream os; + os << std::setw(16) << std::setfill('0') << std::hex << oid.hi; + os << "."; + os << std::setw(16) << std::setfill('0') << std::hex << oid.lo; + + std::unique_ptr impl(new daos_handle_internal_t); + impl->path = coh.impl->path / os.str(); + + impl->path.touch(); + + oh->impl = impl.release(); + return 0; + +} + +int daos_array_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev) { + + ASSERT(oh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (ev != NULL) NOTIMP; + + try { + + oh.impl->path.unlink(); + + } catch (eckit::FailedSystemCall& e) { + + if (oh.impl->path.exists()) throw; + return -DER_NONEXIST; + + } + + return 0; + +} + +int daos_array_open(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, + unsigned int mode, daos_size_t *cell_size, + daos_size_t *chunk_size, daos_handle_t *oh, daos_event_t *ev) { + + ASSERT(coh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (mode != DAOS_OO_RW) NOTIMP; + if (ev != NULL) NOTIMP; + + *cell_size = 1; + *chunk_size = (uint64_t) 1048576; + + std::stringstream os; + os << std::setw(16) << std::setfill('0') << std::hex << oid.hi; + os << "."; + os << std::setw(16) << std::setfill('0') << std::hex << oid.lo; + + std::unique_ptr impl(new daos_handle_internal_t); + impl->path = coh.impl->path / os.str(); + + if (!impl->path.exists()) { + return -DER_NONEXIST; + } + + oh->impl = impl.release(); + return 0; + +} + +int daos_array_open_with_attr(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, + unsigned int mode, daos_size_t cell_size, daos_size_t chunk_size, + daos_handle_t *oh, daos_event_t *ev) { + + ASSERT(coh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (mode != DAOS_OO_RW) NOTIMP; + if (ev != NULL) NOTIMP; + + if (cell_size != 1) NOTIMP; + if (chunk_size != (uint64_t) 1048576) NOTIMP; + + return daos_array_create(coh, oid, th, cell_size, chunk_size, oh, ev); + +} + +int daos_array_close(daos_handle_t oh, daos_event_t *ev) { + + ASSERT(oh.impl); + delete oh.impl; + + if (ev != NULL) NOTIMP; + + return 0; + +} + +// daos_array_write and read have the same limitations and transactional behavior as +// daos_kv_put and get + +int daos_array_write(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, + d_sg_list_t *sgl, daos_event_t *ev) { + + ASSERT(oh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (ev != NULL) NOTIMP; + + if (iod->arr_nr != 1) NOTIMP; + + if (sgl->sg_nr != 1) NOTIMP; + // source memory len + if (sgl->sg_iovs[0].iov_buf_len != sgl->sg_iovs[0].iov_len) NOTIMP; + // source memory vs. target object len + if (sgl->sg_iovs[0].iov_buf_len != iod->arr_rgs[0].rg_len) NOTIMP; + + //sgl->sg_iovs[0].iov_buf is a void * with the data to write + //sgl->sg_iovs[0].iov_buf_len is a size_t with the source len + //iod->arr_rgs[0].rg_idx is a uint64_t with the offset to write from + + eckit::FileHandle fh(oh.impl->path, true); + + //eckit::Length existing_len = fh.size(); + + // if writing data to an already existing and populated file, if the data to write + // is smaller than the file or the data has an offset, the holes will be left with + // pre-existing data (openForAppend) rather than zero-d out (openForWrite) + + fh.openForAppend(eckit::Length(sgl->sg_iovs[0].iov_buf_len)); + eckit::AutoClose closer(fh); + fh.seek(iod->arr_rgs[0].rg_idx); + long res = fh.write(sgl->sg_iovs[0].iov_buf, (long) sgl->sg_iovs[0].iov_buf_len); + ASSERT(res == (long) sgl->sg_iovs[0].iov_buf_len); + + return 0; + +} + +int daos_array_get_size(daos_handle_t oh, daos_handle_t th, daos_size_t *size, + daos_event_t *ev) { + + ASSERT(oh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (ev != NULL) NOTIMP; + + *size = (daos_size_t) oh.impl->path.size(); + + return 0; + +} + +int daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, + d_sg_list_t *sgl, daos_event_t *ev) { + + ASSERT(oh.impl); + if (th.impl != DAOS_TX_NONE.impl) NOTIMP; + if (ev != NULL) NOTIMP; + + if (iod->arr_nr != 1) NOTIMP; + + if (sgl->sg_nr != 1) NOTIMP; + // target memory len + if (sgl->sg_iovs[0].iov_buf_len != sgl->sg_iovs[0].iov_len) NOTIMP; + // target memory vs. source object len + if (sgl->sg_iovs[0].iov_buf_len != iod->arr_rgs[0].rg_len) NOTIMP; + + //sgl->sg_iovs[0].iov_buf is a void * where to read the data into + //iod->arr_rgs[0].rg_len is a size_t with the source size + //iod->arr_rgs[0].rg_idx is a uint64_t with the offset to read from + + eckit::FileHandle fh(oh.impl->path); + eckit::Length len = fh.size(); + if (iod->arr_rgs[0].rg_len + iod->arr_rgs[0].rg_idx > len) return -1; + fh.openForRead(); + eckit::AutoClose closer(fh); + fh.seek(iod->arr_rgs[0].rg_idx); + long res = fh.read(sgl->sg_iovs[0].iov_buf, iod->arr_rgs[0].rg_len); + ASSERT(eckit::Length(res) == eckit::Length(iod->arr_rgs[0].rg_len)); + + return 0; + +} + +int daos_cont_create_snap_opt(daos_handle_t coh, daos_epoch_t *epoch, char *name, + enum daos_snapshot_opts opts, daos_event_t *ev) { + + ASSERT(coh.impl); + if (name != NULL) NOTIMP; + if (opts != (DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT)) NOTIMP; + if (ev != NULL) NOTIMP; + + std::vector files; + std::vector dirs; + + coh.impl->path.children(files, dirs); + + std::string oids = ""; + std::string sep = ""; + + for (std::vector* fileset : {&files, &dirs}) { + for (std::vector::iterator it = fileset->begin(); it != fileset->end(); ++it) { + + std::string oid = it->baseName(); + + if (strstr(oid.c_str(), ".snap")) continue; + + ASSERT(oid.length() == 33); + + oids += sep + oid; + sep = ","; + + } + } + + std::ostringstream os; + os << eckit::TimeStamp("hex"); + std::string ts = os.str(); + ASSERT(ts.length() == 16); + + // if writing data to an already existing and populated file, if the data to write + // is smaller than the file, the holes will be zero-d out (openForWrite) rather than + // left with pre-existing data (openForAppend) + + eckit::FileHandle fh(coh.impl->path / ts + ".snap", true); + fh.openForWrite(eckit::Length(oids.size())); + { + eckit::AutoClose closer(fh); + fh.write(oids.data(), oids.size()); + } + + *epoch = std::stoull(ts, nullptr, 16); + + return 0; + +} + +int daos_cont_destroy_snap(daos_handle_t coh, daos_epoch_range_t epr, + daos_event_t *ev) { + + ASSERT(coh.impl); + if (epr.epr_hi != epr.epr_lo) NOTIMP; + if (ev != NULL) NOTIMP; + + std::stringstream os; + os << std::setw(16) << std::setfill('0') << std::hex << epr.epr_hi; + + eckit::PathName snap = coh.impl->path / os.str() + ".snap"; + + if (!snap.exists()) return -1; + + snap.unlink(); + + return 0; + +} + +int daos_oit_open(daos_handle_t coh, daos_epoch_t epoch, + daos_handle_t *oh, daos_event_t *ev) { + + ASSERT(coh.impl); + if (ev != NULL) NOTIMP; + + std::stringstream os; + os << std::setw(16) << std::setfill('0') << std::hex << epoch; + + std::string ts = os.str(); + + std::unique_ptr impl(new daos_handle_internal_t); + impl->path = coh.impl->path / ts + ".snap"; + + if (!impl->path.exists()) { + return -1; + } + + oh->impl = impl.release(); + return 0; + +} + +int daos_oit_close(daos_handle_t oh, daos_event_t *ev) { + + ASSERT(oh.impl); + delete oh.impl; + + if (ev != NULL) NOTIMP; + + return 0; + +} + +int daos_oit_list(daos_handle_t oh, daos_obj_id_t *oids, uint32_t *oids_nr, + daos_anchor_t *anchor, daos_event_t *ev) { + + static std::vector ongoing_req; + static std::string req_hash; + static unsigned long long n = (((unsigned long long)::getpid()) << 32); + + ASSERT(oh.impl); + if (ev != NULL) NOTIMP; + + if (oids_nr == NULL) return -1; + if (oids == NULL) return -1; + if (anchor == NULL) return -1; + + if (!oh.impl->path.exists()) return -1; + + if (anchor->da_type == DAOS_ANCHOR_TYPE_EOF) return -1; + if (anchor->da_type == DAOS_ANCHOR_TYPE_HKEY) NOTIMP; + + if (anchor->da_type == DAOS_ANCHOR_TYPE_ZERO) { + + /// client process must consume all key names before starting a new request + if (ongoing_req.size() != 0) NOTIMP; + + eckit::Length size(oh.impl->path.size()); + std::vector v(size); + + eckit::FileHandle fh(oh.impl->path, false); + fh.openForRead(); + { + eckit::AutoClose closer(fh); + fh.read(&v[0], size); + } + + std::string oids{v.begin(), v.end()}; + eckit::Tokenizer parse(","); + parse(oids, ongoing_req); + + anchor->da_type = DAOS_ANCHOR_TYPE_KEY; + + std::string hostname = eckit::Main::hostname(); + static std::string format = "%Y%m%d.%H%M%S"; + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + std::string name = os.str(); + while (::access(name.c_str(), F_OK) == 0) { + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + name = os.str(); + } + uuid_t new_uuid = {0}; + eckit::MD5 md5(name); + req_hash = md5.digest(); + + ::memcpy((char*) &(anchor->da_buf[0]), req_hash.c_str(), req_hash.size()); + anchor->da_shard = (uint16_t) req_hash.size(); + + } else { + + if (anchor->da_type != DAOS_ANCHOR_TYPE_KEY) + throw eckit::SeriousBug("Unexpected anchor type"); + + /// different processes cannot collaborate on consuming a same kv_list + /// request (i.e. cannot share a hash) + if (std::string((char*) &(anchor->da_buf[0]), anchor->da_shard) != req_hash) NOTIMP; + + } + + uint32_t remain_oids = *oids_nr; + *oids_nr = 0; + while (remain_oids > 0 && ongoing_req.size() > 0) { + + std::string next_oid = ongoing_req.back(); + + oids[*oids_nr] = daos_obj_id_t{ + std::stoull(next_oid.substr(17, 16), nullptr, 16), + std::stoull(next_oid.substr(0, 16), nullptr, 16) + }; + + ongoing_req.pop_back(); + + remain_oids--; + *oids_nr += 1; + } + + if (ongoing_req.size() == 0) anchor->da_type = DAOS_ANCHOR_TYPE_EOF; + + return 0; + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // extern "C" diff --git a/src/dummy_daos/daos.h b/src/dummy_daos/daos.h new file mode 100644 index 000000000..23c57fc65 --- /dev/null +++ b/src/dummy_daos/daos.h @@ -0,0 +1,329 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * @file daos.h + * @author Nicolau Manubens + * @date Jun 2022 + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +//---------------------------------------------------------------------------------------------------------------------- + +#define DAOS_API_VERSION_MAJOR 2 +#define DAOS_API_VERSION_MINOR 0 +#define DAOS_API_VERSION_FIX 3 + +#define DAOS_PC_RW 0 +#define DAOS_COO_RW 0 +#define DAOS_OO_RW 0 + +/** 32 bits for DAOS internal use */ +#define OID_FMT_INTR_BITS 32 +#define OID_FMT_TYPE_BITS 8 +#define OID_FMT_CLASS_BITS 8 +#define OID_FMT_META_BITS 16 + +#define OID_FMT_TYPE_SHIFT (64 - OID_FMT_TYPE_BITS) +#define OID_FMT_CLASS_SHIFT (OID_FMT_TYPE_SHIFT - OID_FMT_CLASS_BITS) +#define OID_FMT_META_SHIFT (OID_FMT_CLASS_SHIFT - OID_FMT_META_BITS) + +#define OID_FMT_TYPE_MAX ((1ULL << OID_FMT_TYPE_BITS) - 1) +#define OID_FMT_CLASS_MAX ((1ULL << OID_FMT_CLASS_BITS) - 1) +#define OID_FMT_META_MAX ((1ULL << OID_FMT_META_BITS) - 1) + +#define OID_FMT_TYPE_MASK (OID_FMT_TYPE_MAX << OID_FMT_TYPE_SHIFT) +#define OID_FMT_CLASS_MASK (OID_FMT_CLASS_MAX << OID_FMT_CLASS_SHIFT) +#define OID_FMT_META_MASK (OID_FMT_META_MAX << OID_FMT_META_SHIFT) + +#define OC_RESERVED 1 << 30 +#define OC_REDUN_SHIFT 24 +#define OBJ_CLASS_DEF(redun, grp_nr) ((redun << OC_REDUN_SHIFT) | grp_nr) +#define MAX_NUM_GROUPS ((1 << 16UL) - 1) + +#define DAOS_TX_NONE (daos_handle_t){NULL} +#define DAOS_PROP_LABEL_MAX_LEN (127) +#define DAOS_PROP_ENTRIES_MAX_NR (128) + +#define DAOS_ANCHOR_BUF_MAX 104 +#define DAOS_ANCHOR_INIT { \ + .da_type = DAOS_ANCHOR_TYPE_ZERO, \ + .da_shard = 0, \ + .da_flags = 0, \ + .da_sub_anchors = 0, \ + .da_buf = { 0 } } + +#define DER_EXIST 1004 +#define DER_NONEXIST 1005 + +//---------------------------------------------------------------------------------------------------------------------- + +enum daos_otype_t { + DAOS_OT_KV_HASHED = 8, + DAOS_OT_ARRAY = 11, + DAOS_OT_ARRAY_BYTE = 13, +}; + +enum daos_pool_props { + DAOS_PROP_PO_LABEL, + DAOS_PROP_CO_LABEL +}; + +enum daos_snapshot_opts { + DAOS_SNAP_OPT_CR = (1 << 0), + DAOS_SNAP_OPT_OIT = (1 << 1) +}; + +enum daos_obj_redun { + OR_RP_1 = 1 +}; + +enum { + OC_S1 = OBJ_CLASS_DEF(OR_RP_1, 1ULL), + OC_S2 = OBJ_CLASS_DEF(OR_RP_1, 2ULL), + OC_SX = OBJ_CLASS_DEF(OR_RP_1, MAX_NUM_GROUPS) +}; + +#ifdef __cplusplus +extern "C" { +#endif + +struct daos_handle_internal_t; +typedef struct daos_handle_internal_t daos_handle_internal_t; + +typedef struct { + daos_handle_internal_t* impl; +} daos_handle_t; + +/* typedef struct daos_handle_internal_t * daos_handle_t; */ + +typedef uint64_t daos_size_t; + +typedef struct { + uint64_t lo; + uint64_t hi; +} daos_obj_id_t; + +typedef uint32_t daos_oclass_id_t; +typedef uint16_t daos_oclass_hints_t; + +typedef void daos_event_t; +typedef void daos_pool_info_t; +typedef void daos_cont_info_t; + +/* describe object-space target range */ +typedef uint64_t daos_off_t; +typedef struct { + daos_off_t rg_idx; + daos_size_t rg_len; +} daos_range_t; +typedef struct { + daos_size_t arr_nr; + daos_range_t *arr_rgs; + daos_size_t arr_nr_short_read; + daos_size_t arr_nr_read; +} daos_array_iod_t; + +/* describe memory-space target region */ +typedef struct { + void *iov_buf; + size_t iov_buf_len; + size_t iov_len; +} d_iov_t; +typedef struct { + uint32_t sg_nr; + uint32_t sg_nr_out; + d_iov_t *sg_iovs; +} d_sg_list_t; + +/* pool properties */ + +typedef char* d_string_t; + +struct daos_prop_entry { + uint32_t dpe_type; + uint32_t dpe_reserv; + union { + uint64_t dpe_val; + d_string_t dpe_str; + void *dpe_val_ptr; + }; +}; + +typedef struct { + uint32_t dpp_nr; + uint32_t dpp_reserv; + struct daos_prop_entry *dpp_entries; +} daos_prop_t; + +/* cont info */ + +struct daos_pool_cont_info { + uuid_t pci_uuid; + char pci_label[DAOS_PROP_LABEL_MAX_LEN+1]; +}; + +/* kv list */ + +typedef struct { + uint16_t da_type; + uint16_t da_shard; + uint32_t da_flags; + uint64_t da_sub_anchors; + uint8_t da_buf[DAOS_ANCHOR_BUF_MAX]; +} daos_anchor_t; + +typedef struct { + daos_size_t kd_key_len; + uint32_t kd_val_type; +} daos_key_desc_t; + +typedef enum { + DAOS_ANCHOR_TYPE_ZERO = 0, + DAOS_ANCHOR_TYPE_HKEY = 1, + DAOS_ANCHOR_TYPE_KEY = 2, + DAOS_ANCHOR_TYPE_EOF = 3, +} daos_anchor_type_t; + +/* cont list oids */ + +typedef uint64_t daos_epoch_t; + +typedef struct { + daos_epoch_t epr_lo; + daos_epoch_t epr_hi; +} daos_epoch_range_t; + +//---------------------------------------------------------------------------------------------------------------------- + +/* functions */ + +int daos_init(void); + +int daos_fini(void); + +int daos_pool_connect(const char *pool, const char *sys, unsigned int flags, + daos_handle_t *poh, daos_pool_info_t *info, daos_event_t *ev); + +int daos_pool_disconnect(daos_handle_t poh, daos_event_t *ev); + +/* + * warning: the daos_pool_list_cont API call is not fully implemented. Only the pci_label + * member of the info structs is populated in all cases. The pci_uuid member is only + * populated for existing containers which were created without a label. + */ +int daos_pool_list_cont(daos_handle_t poh, daos_size_t *ncont, + struct daos_pool_cont_info *cbuf, daos_event_t *ev); + +int daos_cont_create(daos_handle_t poh, uuid_t *uuid, daos_prop_t *cont_prop, daos_event_t *ev); + +int daos_cont_create_with_label(daos_handle_t poh, const char *label, + daos_prop_t *cont_prop, uuid_t *uuid, + daos_event_t *ev); + +int daos_cont_destroy(daos_handle_t poh, const char *cont, int force, daos_event_t *ev); + +int daos_cont_open(daos_handle_t poh, const char *cont, unsigned int flags, daos_handle_t *coh, + daos_cont_info_t *info, daos_event_t *ev); + +int daos_cont_close(daos_handle_t coh, daos_event_t *ev); + +int daos_cont_alloc_oids(daos_handle_t coh, daos_size_t num_oids, uint64_t *oid, + daos_event_t *ev); + +int daos_obj_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, + enum daos_otype_t type, daos_oclass_id_t cid, + daos_oclass_hints_t hints, uint32_t args); + +int daos_kv_open(daos_handle_t coh, daos_obj_id_t oid, unsigned int mode, + daos_handle_t *oh, daos_event_t *ev); + +int daos_kv_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev); + +int daos_obj_close(daos_handle_t oh, daos_event_t *ev); + +int daos_kv_put(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char *key, + daos_size_t size, const void *buf, daos_event_t *ev); + +int daos_kv_get(daos_handle_t oh, daos_handle_t th, uint64_t flags, const char *key, + daos_size_t *size, void *buf, daos_event_t *ev); + +int daos_kv_remove(daos_handle_t oh, daos_handle_t th, uint64_t flags, + const char *key, daos_event_t *ev); + +int daos_kv_list(daos_handle_t oh, daos_handle_t th, uint32_t *nr, + daos_key_desc_t *kds, d_sg_list_t *sgl, daos_anchor_t *anchor, + daos_event_t *ev); + +static inline bool daos_anchor_is_eof(daos_anchor_t *anchor) { + return anchor->da_type == DAOS_ANCHOR_TYPE_EOF; +} + +int daos_array_generate_oid(daos_handle_t coh, daos_obj_id_t *oid, bool add_attr, daos_oclass_id_t cid, + daos_oclass_hints_t hints, uint32_t args); + +int daos_array_create(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, + daos_size_t cell_size, daos_size_t chunk_size, + daos_handle_t *oh, daos_event_t *ev); + +int daos_array_destroy(daos_handle_t oh, daos_handle_t th, daos_event_t *ev); + +int daos_array_open(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, + unsigned int mode, daos_size_t *cell_size, + daos_size_t *chunk_size, daos_handle_t *oh, daos_event_t *ev); + +int daos_array_open_with_attr(daos_handle_t coh, daos_obj_id_t oid, daos_handle_t th, + unsigned int mode, daos_size_t cell_size, daos_size_t chunk_size, + daos_handle_t *oh, daos_event_t *ev); + +int daos_array_close(daos_handle_t oh, daos_event_t *ev); + +static inline void d_iov_set(d_iov_t *iov, void *buf, size_t size) { + iov->iov_buf = buf; + iov->iov_len = iov->iov_buf_len = size; +} + +int daos_array_write(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, + d_sg_list_t *sgl, daos_event_t *ev); + +int daos_array_get_size(daos_handle_t oh, daos_handle_t th, daos_size_t *size, + daos_event_t *ev); + +int daos_array_read(daos_handle_t oh, daos_handle_t th, daos_array_iod_t *iod, + d_sg_list_t *sgl, daos_event_t *ev); + +int daos_cont_create_snap_opt(daos_handle_t coh, daos_epoch_t *epoch, char *name, + enum daos_snapshot_opts opts, daos_event_t *ev); + +int daos_cont_destroy_snap(daos_handle_t coh, daos_epoch_range_t epr, + daos_event_t *ev); + +int daos_oit_open(daos_handle_t coh, daos_epoch_t epoch, + daos_handle_t *oh, daos_event_t *ev); + +int daos_oit_close(daos_handle_t oh, daos_event_t *ev); + +int daos_oit_list(daos_handle_t oh, daos_obj_id_t *oids, uint32_t *oids_nr, + daos_anchor_t *anchor, daos_event_t *ev); + +//---------------------------------------------------------------------------------------------------------------------- + +#ifdef __cplusplus +} // extern "C" +#endif + diff --git a/src/dummy_daos/daos/tests_lib.cc b/src/dummy_daos/daos/tests_lib.cc new file mode 100644 index 000000000..89595a99f --- /dev/null +++ b/src/dummy_daos/daos/tests_lib.cc @@ -0,0 +1,231 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * @file tests_lib.cc + * @author Nicolau Manubens + * @date Jun 2022 + */ + +#include +#include + +#include "eckit/filesystem/TmpDir.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/log/TimeStamp.h" +#include "eckit/runtime/Main.h" +#include "eckit/utils/MD5.h" + +#include "tests_lib.h" +#include "../dummy_daos.h" + +namespace{ + void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); + }; +} + +//---------------------------------------------------------------------------------------------------------------------- + +extern "C" { + +daos_prop_t* daos_prop_alloc(uint32_t entries_nr) { + + daos_prop_t *prop; + + if (entries_nr > DAOS_PROP_ENTRIES_MAX_NR) { + eckit::Log::error() << "Too many property entries requested."; + return NULL; + } + + D_ALLOC_PTR(prop); + if (prop == NULL) + return NULL; + + if (entries_nr > 0) { + D_ALLOC_ARRAY(prop->dpp_entries, entries_nr); + if (prop->dpp_entries == NULL) { + D_FREE(prop); + return NULL; + } + } + prop->dpp_nr = entries_nr; + return prop; + +} + +void daos_prop_fini(daos_prop_t *prop) { + int i; + + if (!prop->dpp_entries) + goto out; + + for (i = 0; i < prop->dpp_nr; i++) { + struct daos_prop_entry *entry; + + entry = &prop->dpp_entries[i]; + if (entry->dpe_type != DAOS_PROP_PO_LABEL) NOTIMP; + D_FREE(entry->dpe_str); + } + + D_FREE(prop->dpp_entries); +out: + prop->dpp_nr = 0; +} + +void daos_prop_free(daos_prop_t *prop) { + if (prop == NULL) + return; + + daos_prop_fini(prop); + D_FREE(prop); +} + +/// @note: pools are implemented as directories. Upon creation, a new random and unique string +/// is generated, a pool UUID is generated from that string, and a directory is created +/// with the UUID as directory name. If a label is specified, a symlink is created with +/// the label as origin file name and the UUID directory as destination. If no label is +/// specified, no symlink is created. + +int dmg_pool_create(const char *dmg_config_file, + uid_t uid, gid_t gid, const char *grp, + const d_rank_list_t *tgts, + daos_size_t scm_size, daos_size_t nvme_size, + daos_prop_t *prop, + d_rank_list_t *svc, uuid_t uuid) { + + std::string pool_name; + eckit::PathName label_symlink_path; + + if (prop != NULL) { + + if (prop->dpp_nr != 1) NOTIMP; + if (prop->dpp_entries[0].dpe_type != DAOS_PROP_PO_LABEL) NOTIMP; + + struct daos_prop_entry *entry = &prop->dpp_entries[0]; + + if (entry == NULL) NOTIMP; + + pool_name = std::string(entry->dpe_str); + + label_symlink_path = dummy_daos_root() / pool_name; + + if (label_symlink_path.exists()) return -1; + + } + + /// @note: copied from LocalPathName::unique. Ditched StaticMutex as dummy DAOS is not thread safe + + std::string hostname = eckit::Main::hostname(); + + static unsigned long long n = (((unsigned long long)::getpid()) << 32); + + static std::string format = "%Y%m%d.%H%M%S"; + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + + std::string name = os.str(); + + while (::access(name.c_str(), F_OK) == 0) { + std::ostringstream os; + os << eckit::TimeStamp(format) << '.' << hostname << '.' << n++; + name = os.str(); + } + + eckit::MD5 md5(name); + uint64_t hi = std::stoull(md5.digest().substr(0, 8), nullptr, 16); + uint64_t lo = std::stoull(md5.digest().substr(8, 16), nullptr, 16); + ::memcpy(&uuid[0], &hi, sizeof(hi)); + ::memcpy(&uuid[8], &lo, sizeof(lo)); + + char pool_uuid_cstr[37] = ""; + uuid_unparse(uuid, pool_uuid_cstr); + + eckit::PathName pool_path = dummy_daos_root() / pool_uuid_cstr; + + if (pool_path.exists()) throw eckit::SeriousBug("UUID clash in pool create"); + + pool_path.mkdir(); + + if (prop == NULL) return 0; + + if (::symlink(pool_path.path().c_str(), label_symlink_path.path().c_str()) < 0) { + + if (errno == EEXIST) { // link path already exists due to race condition + + deldir(pool_path); + return -1; + + } else { // symlink fails for unknown reason + + throw eckit::FailedSystemCall(std::string("symlink ") + pool_path.path() + " " + label_symlink_path.path()); + + } + + } + + return 0; + +} + +int dmg_pool_destroy(const char *dmg_config_file, + const uuid_t uuid, const char *grp, int force) { + + char uuid_str[37] = ""; + + uuid_unparse(uuid, uuid_str); + + eckit::PathName pool_path = dummy_daos_root() / uuid_str; + + if (!pool_path.exists()) return -1; + + std::vector files; + std::vector dirs; + dummy_daos_root().children(files, dirs); + + /// @note: all pool labels (symlinks) are visited and deleted if they point to the + /// specified pool UUID. None will match if the pool was not labelled. + + for (auto& f : files) { + try { + + if (f.isLink() && f.realName().baseName() == pool_path.baseName()) f.unlink(); + + } catch (eckit::FailedSystemCall& e) { + + if (f.exists()) throw; + + } + } + + deldir(pool_path); + + return 0; + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // extern "C" diff --git a/src/dummy_daos/daos/tests_lib.h b/src/dummy_daos/daos/tests_lib.h new file mode 100644 index 000000000..a6a1680f6 --- /dev/null +++ b/src/dummy_daos/daos/tests_lib.h @@ -0,0 +1,63 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * @file tests_lib.h + * @author Nicolau Manubens + * @date Jun 2022 + */ + +#pragma once + +#include + +#include "../daos.h" + +//---------------------------------------------------------------------------------------------------------------------- + +#define D_ALLOC_ARRAY(ptr, count) (ptr) = (__typeof__(ptr))calloc((count), (sizeof(*ptr))); +#define D_ALLOC_PTR(ptr) D_ALLOC_ARRAY(ptr, 1) +#define D_FREE(ptr) ({ free(ptr); (ptr) = NULL; }) +#define D_STRNDUP(ptr, s, n) (ptr) = strndup(s, n); + +//---------------------------------------------------------------------------------------------------------------------- + +#ifdef __cplusplus +extern "C" { +#endif + +typedef uint32_t d_rank_t; + +typedef struct { + d_rank_t *rl_ranks; + uint32_t rl_nr; +} d_rank_list_t; + +//---------------------------------------------------------------------------------------------------------------------- + +daos_prop_t* daos_prop_alloc(uint32_t entries_nr); + +void daos_prop_free(daos_prop_t *prop); + +int dmg_pool_create(const char *dmg_config_file, + uid_t uid, gid_t gid, const char *grp, + const d_rank_list_t *tgts, + daos_size_t scm_size, daos_size_t nvme_size, + daos_prop_t *prop, + d_rank_list_t *svc, uuid_t uuid); + +int dmg_pool_destroy(const char *dmg_config_file, + const uuid_t uuid, const char *grp, int force); + +//---------------------------------------------------------------------------------------------------------------------- + +#ifdef __cplusplus +} // extern "C" +#endif /* end ifdef __cplusplus */ diff --git a/src/dummy_daos/dummy_daos.cc b/src/dummy_daos/dummy_daos.cc new file mode 100644 index 000000000..cdf8a915a --- /dev/null +++ b/src/dummy_daos/dummy_daos.cc @@ -0,0 +1,47 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * @file dummy_daos.cc + * @author Nicolau Manubens + * @date Jun 2022 + */ + +#include "dummy_daos.h" + +#include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" + +using eckit::PathName; + +//---------------------------------------------------------------------------------------------------------------------- + +typedef struct daos_handle_internal_t { + PathName path; +} daos_handle_internal_t; + +//---------------------------------------------------------------------------------------------------------------------- + +const PathName& dummy_daos_root() { + + static PathName tmpdir = eckit::Resource("$TMPDIR", "/tmp"); + static PathName daos_root = eckit::Resource("$DUMMY_DAOS_DATA_ROOT", tmpdir / "fdb5_dummy_daos"); + return daos_root; + +} + +const PathName& dummy_daos_get_handle_path(daos_handle_t handle) { + + ASSERT(handle.impl); + return handle.impl->path; + +} + +//---------------------------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/dummy_daos/dummy_daos.h b/src/dummy_daos/dummy_daos.h new file mode 100644 index 000000000..51c5d69df --- /dev/null +++ b/src/dummy_daos/dummy_daos.h @@ -0,0 +1,31 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * @file dummy_daos.h + * @author Nicolau Manubens + * @date Jun 2022 + */ + +#pragma once + +#include "daos.h" + +#include "eckit/filesystem/PathName.h" + +using eckit::PathName; + +//---------------------------------------------------------------------------------------------------------------------- + +const PathName& dummy_daos_root(); + +const PathName& dummy_daos_get_handle_path(daos_handle_t handle); + +//---------------------------------------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 147439afd..52f57aa78 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -81,6 +81,8 @@ list( APPEND fdb5_srcs database/Catalogue.h database/DB.cc database/DB.h + database/DatabaseNotFoundException.cc + database/DatabaseNotFoundException.h database/DataStats.cc database/DataStats.h database/DbStats.cc @@ -368,6 +370,60 @@ if( HAVE_RADOSFDB ) ) endif() +if( HAVE_DAOSFDB ) + list( APPEND fdb5_srcs + daos/DaosOID.cc + daos/DaosOID.h + daos/DaosName.cc + daos/DaosName.h + daos/DaosObject.cc + daos/DaosObject.h + daos/DaosContainer.cc + daos/DaosContainer.h + daos/DaosPool.cc + daos/DaosPool.h + daos/DaosSession.cc + daos/DaosSession.h + daos/DaosArrayHandle.cc + daos/DaosArrayHandle.h + daos/DaosKeyValueHandle.cc + daos/DaosKeyValueHandle.h + daos/DaosStore.h + daos/DaosStore.cc + daos/DaosFieldLocation.h + daos/DaosFieldLocation.cc + daos/DaosException.h + daos/DaosException.cc + daos/DaosEngine.h + daos/DaosEngine.cc + daos/DaosCatalogue.h + daos/DaosCatalogue.cc + daos/DaosCatalogueWriter.h + daos/DaosCatalogueWriter.cc + daos/DaosCatalogueReader.h + daos/DaosCatalogueReader.cc + daos/DaosIndex.h + daos/DaosIndex.cc + daos/DaosIndexLocation.h + daos/DaosIndexLocation.cc + daos/DaosCommon.h + daos/DaosCommon.cc + daos/DaosWipeVisitor.h + daos/DaosWipeVisitor.cc + daos/DaosArrayPartHandle.h + daos/DaosArrayPartHandle.cc + daos/DaosLazyFieldLocation.h + daos/DaosLazyFieldLocation.cc + daos/UUID.h + daos/UUID.cc + ) +else() + set( DAOS_LIBRARIES "" ) + set( DAOS_INCLUDE_DIRS "" ) + set( UUID_LIBRARIES "" ) + set( UUID_INCLUDE_DIRS "" ) +endif() + ecbuild_add_library( TARGET fdb5 @@ -390,15 +446,20 @@ ecbuild_add_library( metkit eckit eckit_option + "${UUID_LIBRARIES}" PRIVATE_INCLUDES "${PMEM_INCLUDE_DIRS}" "${LUSTREAPI_INCLUDE_DIRS}" + "${DAOS_INCLUDE_DIRS}" + "${DAOS_TESTS_INCLUDE_DIRS}" PRIVATE_LIBS ${grib_handling_pkg} ${PMEM_LIBRARIES} ${LUSTREAPI_LIBRARIES} + "${DAOS_LIBRARIES}" + "${DAOS_TESTS_LIBRARIES}" ) if(HAVE_FDB_BUILD_TOOLS) diff --git a/src/fdb5/LibFdb5.cc b/src/fdb5/LibFdb5.cc index 6bbe46d3e..ec688c31e 100644 --- a/src/fdb5/LibFdb5.cc +++ b/src/fdb5/LibFdb5.cc @@ -40,10 +40,10 @@ LibFdb5& LibFdb5::instance() { return libfdb; } -const Config& LibFdb5::defaultConfig() { +const Config& LibFdb5::defaultConfig(const eckit::Configuration& userConfig) { if(!config_) { Config cfg; - config_.reset( new Config( std::move(cfg.expandConfig()) ) ); + config_.reset( new Config( std::move(cfg.expandConfig()), userConfig ) ); } return *config_; } diff --git a/src/fdb5/LibFdb5.h b/src/fdb5/LibFdb5.h index 23500f452..524c93b9c 100644 --- a/src/fdb5/LibFdb5.h +++ b/src/fdb5/LibFdb5.h @@ -71,7 +71,7 @@ class LibFdb5 : public eckit::system::Library { RemoteProtocolVersion remoteProtocolVersion() const; /// Returns the default configuration according to the rules of FDB configuration search - const Config& defaultConfig(); + const Config& defaultConfig(const eckit::Configuration& userConfig = eckit::LocalConfiguration()); bool dontDeregisterFactories() const; diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc index 15ca1bb2f..811013899 100644 --- a/src/fdb5/api/helpers/ListIterator.cc +++ b/src/fdb5/api/helpers/ListIterator.cc @@ -41,20 +41,16 @@ Key ListElement::combinedKey() const { return combined; } -void ListElement::print(std::ostream &out, bool withLocation, bool withLength) const { +void ListElement::print(std::ostream &out, bool withLocation, bool withLength, bool withTimestamp, const char* sep) const { if (!withLocation && location_ && !location_->host().empty()) { out << "host=" << location_->host() << ","; } for (const auto& bit : keyParts_) { out << bit; } - if (location_) { - if (withLocation) { - out << " " << *location_; - } else if (withLength) { - out << ",length=" << location_->length(); - } - } + if (location_ && withLocation) out << sep << *location_; + if (withLength) out << sep << "length=" << location_->length(); + if (withTimestamp) out << sep << "timestamp=" << timestamp_; } void ListElement::json(eckit::JSON& json) const { diff --git a/src/fdb5/api/helpers/ListIterator.h b/src/fdb5/api/helpers/ListIterator.h index 255fc8450..79788d0ef 100644 --- a/src/fdb5/api/helpers/ListIterator.h +++ b/src/fdb5/api/helpers/ListIterator.h @@ -54,7 +54,7 @@ class ListElement { Key combinedKey() const; - void print(std::ostream& out, bool withLocation=false, bool withLength=false) const; + void print(std::ostream& out, bool withLocation=false, bool withLength=false, bool withTimestamp=false, const char* sep = " ") const; void json(eckit::JSON& json) const; private: // methods @@ -102,6 +102,13 @@ class ListIterator : public APIIterator { ListIterator(ListIterator&& iter) : APIIterator(std::move(iter)), seenKeys_(std::move(iter.seenKeys_)), deduplicate_(iter.deduplicate_) {} + ListIterator& operator=(ListIterator&& iter) { + seenKeys_ = std::move(iter.seenKeys_); + deduplicate_ = iter.deduplicate_; + APIIterator::operator=(std::move(iter)); + return *this; + } + bool next(ListElement& elem) { ListElement tmp; while (APIIterator::next(tmp)) { diff --git a/src/fdb5/api/local/WipeVisitor.h b/src/fdb5/api/local/WipeVisitor.h index 6cd6db2a4..f3c56a186 100644 --- a/src/fdb5/api/local/WipeVisitor.h +++ b/src/fdb5/api/local/WipeVisitor.h @@ -53,6 +53,8 @@ class WipeVisitor : public QueryVisitor { void visitDatum(const Field&, const Key&) override { NOTIMP; } void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; } + virtual void onDatabaseNotFound(const fdb5::DatabaseNotFoundException& e) override { throw e; } + private: // members eckit::Channel out_; diff --git a/src/fdb5/daos/DaosArrayHandle.cc b/src/fdb5/daos/DaosArrayHandle.cc new file mode 100644 index 000000000..5cde01e8a --- /dev/null +++ b/src/fdb5/daos/DaosArrayHandle.cc @@ -0,0 +1,184 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "eckit/exception/Exceptions.h" + +#include "fdb5/daos/DaosArrayHandle.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosException.h" + +using eckit::Length; +using eckit::Offset; + +namespace fdb5 { + +DaosArrayHandle::DaosArrayHandle(const fdb5::DaosArrayName& name) : name_(name), open_(false), offset_(0) {} + +DaosArrayHandle::~DaosArrayHandle() { + + if (open_) eckit::Log::error() << "DaosArrayHandle not closed before destruction." << std::endl; + +} + +void DaosArrayHandle::print(std::ostream& s) const { + s << "DaosArrayHandle[notimp]"; +} + +void DaosArrayHandle::openForWrite(const Length& len) { + + if (open_) throw eckit::SeriousBug{"Handle already opened."}; + + session(); + + fdb5::DaosPool& p = session_->getPool(name_.poolName()); + fdb5::DaosContainer& c = p.ensureContainer(name_.containerName()); + + /// @note: to open/create an array without generating a snapshot, we must: + /// - attempt array open and check if rc is 0 or DER_NONEXIST (DaosArray(session, name)) + /// - attempt array create and check if rc is 0 or DER_EXIST (c.createArray(oid)) + /// we do the latter first because it is the most likely to succeed. + /// If the operation fails, the DAOS client will generate an ERR log and usually persist + /// it to a log file (potential performance impact). So we want to have as few of these + /// failures as possible. + /// @todo: implement DaosContainer::ensureArray, which attempts createArray with a catch+dismiss? + /// @todo: have dummy daos_create_array return DER_EXIST where relevant. + /// This would probably require breaking transactionality of dummy + /// daos_create_array, and/or hit its performance. + try { + arr_.emplace( c.createArray(name_.OID()) ); + } catch (fdb5::DaosEntityAlreadyExistsException& e) { + arr_.emplace( session_.value(), name_ ); + } + + arr_->open(); + + /// @todo: should wipe object content? + + open_ = true; + + offset_ = eckit::Offset(0); + +} + +/// @note: the array size is retrieved here and ::read. For a more optimised reading +/// if the size is known in advance, see DaosArrayPartHandle. +Length DaosArrayHandle::openForRead() { + + if (open_) throw eckit::SeriousBug{"Handle already opened."}; + + session(); + + arr_.emplace(session_.value(), name_); + + arr_->open(); + + open_ = true; + + return size(); + +} + +long DaosArrayHandle::write(const void* buf, long len) { + + ASSERT(open_); + + long written = arr_->write(buf, len, offset_); + + offset_ += written; + + return written; + +} + +long DaosArrayHandle::read(void* buf, long len) { + + ASSERT(open_); + + long read = arr_->read(buf, len, offset_); + + /// @note: if the buffer is oversized, daos does not return the actual smaller size read, + /// so it is calculated here and returned to the user as expected + eckit::Length s = size(); + if (len > s - offset_) read = s - offset_; + + offset_ += read; + + return read; + +} + +void DaosArrayHandle::close() { + + if (!open_) return; + + arr_->close(); + + open_ = false; + +} + +void DaosArrayHandle::flush() { + + /// empty implmenetation + +} + +Length DaosArrayHandle::size() { + + return Length(name_.size()); + +} + +Length DaosArrayHandle::estimate() { + + return size(); + +} + +Offset DaosArrayHandle::position() { + + /// @todo: should position() crash if unopened? + return offset_; + +} + +Offset DaosArrayHandle::seek(const Offset& offset) { + + offset_ = offset; + /// @todo: assert offset <= size() ? + return offset_; + +} + +bool DaosArrayHandle::canSeek() const { + + return true; + +} + +std::string DaosArrayHandle::title() const { + + return name_.asString(); + +} + +fdb5::DaosSession& DaosArrayHandle::session() { + + if (!session_.has_value()) session_.emplace(); + return session_.value(); + +} + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosArrayHandle.h b/src/fdb5/daos/DaosArrayHandle.h new file mode 100644 index 000000000..f0bc752e2 --- /dev/null +++ b/src/fdb5/daos/DaosArrayHandle.h @@ -0,0 +1,86 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jul 2022 + +#pragma once + +#include "eckit/io/DataHandle.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosName.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosArray; + +class DaosArrayName; + +class DaosArrayHandle : public eckit::DataHandle { + +public: // methods + + DaosArrayHandle(const fdb5::DaosArrayName&); + + ~DaosArrayHandle(); + + virtual void print(std::ostream&) const override; + + virtual void openForWrite(const eckit::Length&) override; + virtual void openForAppend(const eckit::Length&) override { NOTIMP; }; + virtual eckit::Length openForRead() override; + + virtual long write(const void*, long) override; + virtual long read(void*, long) override; + virtual void close() override; + virtual void flush() override; + + virtual eckit::Length size() override; + virtual eckit::Length estimate() override; + virtual eckit::Offset position() override; + virtual eckit::Offset seek(const eckit::Offset&) override; + virtual bool canSeek() const override; + // virtual void skip(const eckit::Length&) override; + + // virtual void rewind() override; + // virtual void restartReadFrom(const Offset&) override; + // virtual void restartWriteFrom(const Offset&) override; + + virtual std::string title() const override; + + // virtual void encode(Stream&) const override; + // virtual const ReanimatorBase& reanimator() const override { return reanimator_; } + + // static const ClassSpec& classSpec() { return classSpec_; } + +private: // methods + + fdb5::DaosSession& session(); + +private: // members + + // mutable because title() calls DaosArrayName::asString which may update (generate) OID + mutable fdb5::DaosArrayName name_; + std::optional session_; + std::optional arr_; + bool open_; + eckit::Offset offset_; + + // static ClassSpec classSpec_; + // static Reanimator reanimator_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosArrayPartHandle.cc b/src/fdb5/daos/DaosArrayPartHandle.cc new file mode 100644 index 000000000..5c5ba83a9 --- /dev/null +++ b/src/fdb5/daos/DaosArrayPartHandle.cc @@ -0,0 +1,142 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "eckit/exception/Exceptions.h" + +#include "fdb5/daos/DaosArrayPartHandle.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosException.h" + +using eckit::Length; +using eckit::Offset; + +namespace fdb5 { + +DaosArrayPartHandle::DaosArrayPartHandle(const fdb5::DaosArrayName& name, + const eckit::Offset& off, + const eckit::Length& len) : name_(name), open_(false), offset_(off), len_(len) {} + +DaosArrayPartHandle::~DaosArrayPartHandle() { + + if (open_) eckit::Log::error() << "DaosArrayPartHandle not closed before destruction." << std::endl; + +} + +void DaosArrayPartHandle::print(std::ostream& s) const { + s << "DaosArrayPartHandle[notimp]"; +} + +Length DaosArrayPartHandle::openForRead() { + + if (open_) throw eckit::SeriousBug{"Handle already opened."}; + + session(); + + arr_.emplace(session_.value(), name_); + + arr_->open(); + + open_ = true; + + return size(); + +} + +long DaosArrayPartHandle::read(void* buf, long len) { + + ASSERT(open_); + + /// @note: if the buffer is oversized, daos does not return the actual smaller size read, + /// so it is calculated here and returned to the user as expected + eckit::Length s = size(); + if (len > s - offset_) len = s - offset_; + + long read = arr_->read(buf, len, offset_); + + offset_ += read; + + return read; + +} + +void DaosArrayPartHandle::close() { + + if (!open_) return; + + arr_->close(); + + open_ = false; + +} + +void DaosArrayPartHandle::flush() { + + /// empty implmenetation + +} + +Length DaosArrayPartHandle::size() { + + return len_; + +} + +Length DaosArrayPartHandle::estimate() { + + return size(); + +} + +Offset DaosArrayPartHandle::position() { + + /// @todo: should position() crash if unopened? + return offset_; + +} + +Offset DaosArrayPartHandle::seek(const Offset& offset) { + + offset_ = offset; + /// @todo: assert offset <= size() ? + return offset_; + +} + +bool DaosArrayPartHandle::canSeek() const { + + return true; + +} + +// void DaosArrayHandle::skip(const Length& len) { + +// offset_ += Offset(len); + +// } + +std::string DaosArrayPartHandle::title() const { + + return name_.asString(); + +} + +fdb5::DaosSession& DaosArrayPartHandle::session() { + + if (!session_.has_value()) session_.emplace(); + return session_.value(); + +} + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosArrayPartHandle.h b/src/fdb5/daos/DaosArrayPartHandle.h new file mode 100644 index 000000000..e82457390 --- /dev/null +++ b/src/fdb5/daos/DaosArrayPartHandle.h @@ -0,0 +1,92 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Oct 2023 + +#pragma once + +#include "eckit/io/DataHandle.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosName.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosArray; + +class DaosArrayName; + +/// @note: because daos_array_read does not report actual amount of bytes read, +/// DaosArrayHandle::read must check the actual size and returns it if smaller +/// than the provided buffer, which reduces performance. DaosArrayPartHandle +/// circumvents that check for cases where the object size is known. +/// @note: see DaosArray::read +class DaosArrayPartHandle : public eckit::DataHandle { + +public: // methods + + DaosArrayPartHandle(const fdb5::DaosArrayName&, const eckit::Offset&, const eckit::Length&); + + ~DaosArrayPartHandle(); + + virtual void print(std::ostream&) const override; + + // virtual void openForWrite(const eckit::Length&) override; + // virtual void openForAppend(const eckit::Length&) override; + virtual eckit::Length openForRead() override; + + // virtual long write(const void*, long) override; + virtual long read(void*, long) override; + virtual void close() override; + virtual void flush() override; + + virtual eckit::Length size() override; + virtual eckit::Length estimate() override; + virtual eckit::Offset position() override; + virtual eckit::Offset seek(const eckit::Offset&) override; + virtual bool canSeek() const override; + // virtual void skip(const eckit::Length&) override; + + // virtual void rewind() override; + // virtual void restartReadFrom(const Offset&) override; + // virtual void restartWriteFrom(const Offset&) override; + + virtual std::string title() const override; + + // virtual void encode(Stream&) const override; + // virtual const ReanimatorBase& reanimator() const override { return reanimator_; } + + // static const ClassSpec& classSpec() { return classSpec_; } + +private: // methods + + fdb5::DaosSession& session(); + +private: // members + + // mutable because title() calls DaosArrayName::asString which may update (generate) OID + mutable fdb5::DaosArrayName name_; + std::optional session_; + std::optional arr_; + bool open_; + eckit::Offset offset_; + eckit::Length len_; + + // static ClassSpec classSpec_; + // static Reanimator reanimator_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosCatalogue.cc b/src/fdb5/daos/DaosCatalogue.cc new file mode 100644 index 000000000..137a170b7 --- /dev/null +++ b/src/fdb5/daos/DaosCatalogue.cc @@ -0,0 +1,184 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/config/Resource.h" +#include "eckit/serialisation/MemoryStream.h" + +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/LibFdb5.h" +#include "fdb5/database/DatabaseNotFoundException.h" + +#include "fdb5/daos/DaosCatalogue.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosIndex.h" +#include "fdb5/daos/DaosWipeVisitor.h" + +// using namespace eckit; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosCatalogue::DaosCatalogue(const Key& key, const fdb5::Config& config) : + Catalogue(key, ControlIdentifiers{}, config), DaosCommon(config, "catalogue", key) { + + // TODO: apply the mechanism in RootManager::directory, using + // FileSpaceTables to determine root_pool_name_ according to key + // and using DbPathNamerTables to determine db_cont_name_ according + // to key + +} + +DaosCatalogue::DaosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config) : + Catalogue(Key(), controlIdentifiers, config), DaosCommon(config, "catalogue", uri) { + + // Read the real DB key into the DB base object + try { + + fdb5::DaosSession s{}; + fdb5::DaosKeyValueName n{pool_, db_cont_, catalogue_kv_}; + fdb5::DaosKeyValue db_kv{s, n}; + + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + dbKey_ = fdb5::Key(ms); + + } catch (fdb5::DaosEntityNotFoundException& e) { + + throw fdb5::DatabaseNotFoundException( + std::string("DaosCatalogue database not found ") + + "(pool: '" + pool_ + "', container: '" + db_cont_ + "')" + ); + + } + +} + +bool DaosCatalogue::exists() const { + + fdb5::DaosKeyValueName catalogue_kv_name{pool_, db_cont_, catalogue_kv_}; + return catalogue_kv_name.exists(); + +} + +eckit::URI DaosCatalogue::uri() const { + + return fdb5::DaosName{db_kv_->poolName(), db_kv_->containerName()}.URI(); + +} + +const Schema& DaosCatalogue::schema() const { + + return schema_; + +} + +void DaosCatalogue::loadSchema() { + + eckit::Timer timer("DaosCatalogue::loadSchema()", eckit::Log::debug()); + + /// @note: performed RPCs: + /// - daos_obj_generate_oid + /// - daos_kv_open + /// - daos_kv_get without a buffer + /// - daos_kv_get + fdb5::DaosKeyValueName nkv{pool_, db_cont_, catalogue_kv_}; + fdb5::DaosSession s{}; + fdb5::DaosKeyValue kv{s, nkv}; + uint64_t size = kv.size("schema"); + std::vector v(size); + kv.get("schema", v.data(), size); + + std::istringstream stream{std::string(v.begin(), v.end())}; + schema_.load(stream); + +} + +WipeVisitor* DaosCatalogue::wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const { + return new DaosWipeVisitor(*this, store, request, out, doit, porcelain, unsafeWipeAll); +} + +std::vector DaosCatalogue::indexes(bool) const { + + /// @note: sorted is not implemented as is not necessary in this backend. + + fdb5::DaosKeyValueName catalogue_kv_name{pool_, db_cont_, catalogue_kv_}; + fdb5::DaosSession s{}; + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db kv list keys (daos_kv_list) + fdb5::DaosKeyValue catalogue_kv{s, catalogue_kv_name}; /// @note: throws if not exists + + std::vector res; + + for (const auto& key : catalogue_kv.keys()) { + + /// @todo: document these well. Single source these reserved values. + /// Ensure where appropriate that user-provided keys do not collide. + if (key == "schema" || key == "key") continue; + + /// @note: performed RPCs: + /// - db kv get index location size (daos_kv_get without a buffer) + /// - db kv get index location (daos_kv_get) + uint64_t size{catalogue_kv.size(key)}; + std::vector v(size); + catalogue_kv.get(key, v.data(), size); + + fdb5::DaosKeyValueName index_kv_name{eckit::URI(std::string(v.begin(), v.end()))}; + + /// @note: performed RPCs: + /// - index kv open (daos_kv_open) + /// - index kv get size (daos_kv_get without a buffer) + /// - index kv get key (daos_kv_get) + /// @note: the following three lines intend to check whether the index kv exists + /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, + /// so it is not useful on its own to check whether the index KV existed or not. + /// Instead, presence of a "key" key in the KV is used to determine if the index + /// KV existed. + fdb5::DaosKeyValue index_kv{s, index_kv_name}; + std::optional index_key; + try { + std::vector data; + eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index KV"); + index_key.emplace(ms); + } catch (fdb5::DaosEntityNotFoundException& e) { + continue; /// @note: the index_kv may not exist after a failed wipe + /// @todo: the index_kv may exist even if it does not have the "key" key + } + + res.push_back(Index(new fdb5::DaosIndex(index_key.value(), index_kv_name, false))); + + } + + return res; + +} + +std::string DaosCatalogue::type() const { + + return DaosCatalogue::catalogueTypeName(); + +} + +void DaosCatalogue::remove(const fdb5::DaosNameBase& n, std::ostream& logAlways, std::ostream& logVerbose, bool doit) { + + ASSERT(n.hasContainerName()); + + logVerbose << "Removing " << (n.hasOID() ? "KV" : "container") << ": "; + logAlways << n.URI() << std::endl; + if (doit) n.destroy(); + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosCatalogue.h b/src/fdb5/daos/DaosCatalogue.h new file mode 100644 index 000000000..3cfdae898 --- /dev/null +++ b/src/fdb5/daos/DaosCatalogue.h @@ -0,0 +1,78 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Feb 2022 + +#pragma once + +#include "fdb5/database/DB.h" +#include "fdb5/rules/Schema.h" +#include "fdb5/daos/DaosCommon.h" +#include "fdb5/daos/DaosEngine.h" +#include "fdb5/daos/DaosOID.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB that implements the FDB on DAOS + +class DaosCatalogue : public Catalogue, public DaosCommon { + +public: // methods + + DaosCatalogue(const Key& key, const fdb5::Config& config); + DaosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config); + + static const char* catalogueTypeName() { return fdb5::DaosEngine::typeName(); } + + eckit::URI uri() const override; + const Key& indexKey() const override { return currentIndexKey_; } + + static void remove(const fdb5::DaosNameBase&, std::ostream& logAlways, std::ostream& logVerbose, bool doit); + + std::string type() const override; + + void checkUID() const override { NOTIMP; }; + bool exists() const override; + void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override { NOTIMP; }; + std::vector metadataPaths() const override { NOTIMP; }; + const Schema& schema() const override; + + StatsReportVisitor* statsReportVisitor() const override { NOTIMP; }; + PurgeVisitor* purgeVisitor(const Store& store) const override { NOTIMP; }; + WipeVisitor* wipeVisitor(const Store& store, const metkit::mars::MarsRequest& request, std::ostream& out, bool doit, bool porcelain, bool unsafeWipeAll) const override; + MoveVisitor* moveVisitor(const Store& store, const metkit::mars::MarsRequest& request, const eckit::URI& dest, eckit::Queue& queue) const override { NOTIMP; }; + void maskIndexEntry(const Index& index) const override { NOTIMP; }; + + void loadSchema() override; + + std::vector indexes(bool sorted=false) const override; + + void allMasked(std::set>& metadata, + std::set& data) const override { NOTIMP; }; + + // Control access properties of the DB + void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override { NOTIMP; }; + +protected: // members + + Key currentIndexKey_; + +private: // members + + Schema schema_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosCatalogueReader.cc b/src/fdb5/daos/DaosCatalogueReader.cc new file mode 100644 index 000000000..f56da8ccc --- /dev/null +++ b/src/fdb5/daos/DaosCatalogueReader.cc @@ -0,0 +1,140 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/LibFdb5.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosName.h" + +#include "fdb5/daos/DaosIndex.h" +#include "fdb5/daos/DaosCatalogueReader.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// @note: as opposed to the TOC catalogue, the DAOS catalogue does not pre-load all indexes from storage. +/// Instead, it selects and loads only those indexes that are required to fulfil the request. + +DaosCatalogueReader::DaosCatalogueReader(const Key& key, const fdb5::Config& config) : + DaosCatalogue(key, config) { + + /// @todo: schema is being loaded at DaosCatalogueWriter creation for write, but being loaded + /// at DaosCatalogueReader::open for read. Is this OK? + +} + +DaosCatalogueReader::DaosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config) : + DaosCatalogue(uri, ControlIdentifiers{}, config) {} + +bool DaosCatalogueReader::selectIndex(const Key &key) { + + if (currentIndexKey_ == key) { + return true; + } + + /// @todo: shouldn't this be set only if found a matching index? + currentIndexKey_ = key; + + if (indexes_.find(key) == indexes_.end()) { + + fdb5::DaosKeyValueName catalogue_kv{pool_, db_cont_, catalogue_kv_}; + + fdb5::DaosSession s{}; + + /// @note: performed RPCs: + /// - generate catalogue kv oid (daos_obj_generate_oid) + /// - ensure catalogue kv exists (daos_kv_open) + fdb5::DaosKeyValue catalogue_kv_obj{s, catalogue_kv}; + + int idx_loc_max_len = 512; /// @todo: take from config + std::vector n((long) idx_loc_max_len); + long res; + + try { + + /// @note: performed RPCs: + /// - retrieve index kv location from catalogue kv (daos_kv_get) + res = catalogue_kv_obj.get(key.valuesToString(), &n[0], idx_loc_max_len); + + } catch (fdb5::DaosEntityNotFoundException& e) { + + /// @note: performed RPCs: + /// - close catalogue kv (daos_obj_close) + + return false; + + } + + fdb5::DaosKeyValueName index_kv{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}; + + indexes_[key] = Index(new fdb5::DaosIndex(key, index_kv, true)); + + /// @note: performed RPCs: + /// - close catalogue kv (daos_obj_close) + + } + + current_ = indexes_[key]; + + return true; + +} + +void DaosCatalogueReader::deselectIndex() { + + NOTIMP; //< should not be called + +} + +bool DaosCatalogueReader::open() { + + /// @note: performed RPCs: + /// - daos_pool_connect + /// - daos_cont_open + /// - daos_obj_generate_oid + /// - daos_kv_open + if (!DaosCatalogue::exists()) { + return false; + } + + DaosCatalogue::loadSchema(); + return true; + +} + +bool DaosCatalogueReader::axis(const std::string &keyword, eckit::StringSet &s) const { + + bool found = false; + if (current_.axes().has(keyword)) { + found = true; + const eckit::DenseSet& a = current_.axes().values(keyword); + s.insert(a.begin(), a.end()); + } + return found; + +} + +bool DaosCatalogueReader::retrieve(const Key& key, Field& field) const { + + eckit::Log::debug() << "Trying to retrieve key " << key << std::endl; + eckit::Log::debug() << "Scanning index " << current_.location() << std::endl; + + if (!current_.mayContain(key)) return false; + + return current_.get(key, fdb5::Key(), field); + +} + +static fdb5::CatalogueBuilder builder("daos.reader"); + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosCatalogueReader.h b/src/fdb5/daos/DaosCatalogueReader.h new file mode 100644 index 000000000..25618c6d5 --- /dev/null +++ b/src/fdb5/daos/DaosCatalogueReader.h @@ -0,0 +1,60 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Mar 2023 + +#pragma once + +#include "fdb5/daos/DaosCatalogue.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB that implements the FDB on DAOS + +class DaosCatalogueReader : public DaosCatalogue, public CatalogueReader { + +public: // methods + + DaosCatalogueReader(const Key& key, const fdb5::Config& config); + DaosCatalogueReader(const eckit::URI& uri, const fdb5::Config& config); + + DbStats stats() const override { NOTIMP; } + + bool selectIndex(const Key &key) override; + void deselectIndex() override; + + bool open() override; + void flush() override {} + void clean() override {} + void close() override {} + + bool axis(const std::string &keyword, eckit::StringSet &s) const override; + + bool retrieve(const Key& key, Field& field) const override; + + void print( std::ostream &out ) const override { NOTIMP; } + +private: // types + + typedef std::map< Key, Index> IndexStore; + +private: // members + + IndexStore indexes_; + Index current_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosCatalogueWriter.cc b/src/fdb5/daos/DaosCatalogueWriter.cc new file mode 100644 index 000000000..3a9f0b661 --- /dev/null +++ b/src/fdb5/daos/DaosCatalogueWriter.cc @@ -0,0 +1,353 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include +#include + +#include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/serialisation/HandleStream.h" + +#include "fdb5/LibFdb5.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosKeyValueHandle.h" + +#include "fdb5/daos/DaosIndex.h" +#include "fdb5/daos/DaosCatalogueWriter.h" + +// using namespace eckit; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosCatalogueWriter::DaosCatalogueWriter(const Key &key, const fdb5::Config& config) : + DaosCatalogue(key, config), firstIndexWrite_(false) { + + + fdb5::DaosSession s{}; + + /// @note: performed RPCs: + /// - daos_pool_connect + /// - root cont open (daos_cont_open) + /// - root cont create (daos_cont_create) + fdb5::DaosPool& p = s.getPool(pool_); + p.ensureContainer(root_cont_); + + fdb5::DaosKeyValueName main_kv_name{pool_, root_cont_, main_kv_}; + + /// @note: the DaosKeyValue constructor checks if the kv exists, which results in creation if not exists + /// @note: performed RPCs: + /// - main kv open (daos_kv_open) + fdb5::DaosKeyValue main_kv{s, main_kv_name}; + + fdb5::DaosKeyValueName catalogue_kv_name{pool_, db_cont_, catalogue_kv_}; + + /// @note: performed RPCs: + /// - check if main kv contains db key (daos_kv_get without a buffer) + if (!main_kv.has(db_cont_)) { + + /// create catalogue kv + catalogue_kv_name.create(); + + /// write schema under "schema" + eckit::Log::debug() << "Copy schema from " + << config_.schemaPath() + << " to " + << catalogue_kv_name.URI().asString() + << " at key 'schema'." + << std::endl; + + eckit::FileHandle in(config_.schemaPath()); + std::unique_ptr out(catalogue_kv_name.dataHandle("schema")); + in.copyTo(*out); + + /// write dbKey under "key" + eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + hs << dbKey_; + } + + int db_key_max_len = 512; // @todo: take from config + if (hs.bytesWritten() > db_key_max_len) + throw eckit::Exception("Serialised db key exceeded configured maximum db key length."); + + fdb5::DaosKeyValue{s, catalogue_kv_name}.put("key", h.data(), hs.bytesWritten()); + + /// index newly created catalogue kv in main kv + int db_loc_max_len = 512; // @todo: take from config + std::string nstr = catalogue_kv_name.URI().asString(); + if (nstr.length() > db_loc_max_len) + throw eckit::Exception("Serialised db location exceeded configured maximum db location length."); + + main_kv.put(db_cont_, nstr.data(), nstr.length()); + + } + + /// @todo: record or read dbUID + + /// @note: performed RPCs: + /// - catalogue container open (daos_cont_open) + /// - get schema from catalogue kv (daos_kv_get) + DaosCatalogue::loadSchema(); + + /// @todo: TocCatalogue::checkUID(); + +} + +DaosCatalogueWriter::DaosCatalogueWriter(const eckit::URI &uri, const fdb5::Config& config) : + DaosCatalogue(uri, ControlIdentifiers{}, config), firstIndexWrite_(false) { + + NOTIMP; + +} + +DaosCatalogueWriter::~DaosCatalogueWriter() { + + clean(); + close(); + +} + +bool DaosCatalogueWriter::selectIndex(const Key& key) { + + currentIndexKey_ = key; + + if (indexes_.find(key) == indexes_.end()) { + + fdb5::DaosKeyValueName catalogue_kv{pool_, db_cont_, catalogue_kv_}; + + fdb5::DaosSession s{}; + + /// @note: performed RPCs: + /// - generate catalogue kv oid (daos_obj_generate_oid) + /// - ensure catalogue kv exists (daos_kv_open) + fdb5::DaosKeyValue catalogue_kv_obj{s, catalogue_kv}; + + int idx_loc_max_len = 512; /// @todo: take from config + + try { + + std::vector n((long) idx_loc_max_len); + long res; + + /// @note: performed RPCs: + /// - get index location from catalogue kv (daos_kv_get) + res = catalogue_kv_obj.get(key.valuesToString(), &n[0], idx_loc_max_len); + + indexes_[key] = Index( + new fdb5::DaosIndex( + key, + fdb5::DaosKeyValueName{eckit::URI{std::string{n.begin(), std::next(n.begin(), res)}}}, + false + ) + ); + + } catch (fdb5::DaosEntityNotFoundException& e) { + + firstIndexWrite_ = true; + + indexes_[key] = Index( + new fdb5::DaosIndex( + key, + fdb5::DaosName{pool_, db_cont_} + ) + ); + + /// index index kv in catalogue kv + std::string nstr{indexes_[key].location().uri().asString()}; + if (nstr.length() > idx_loc_max_len) + throw eckit::Exception("Serialised index location exceeded configured maximum index location length."); + /// @note: performed RPCs (only if the index wasn't visited yet and index kv doesn't exist yet, i.e. only on first write to an index key): + /// - record index kv location into catalogue kv (daos_kv_put) -- always performed + catalogue_kv_obj.put(key.valuesToString(), nstr.data(), nstr.length()); + + /// @note: performed RPCs: + /// - close index kv when destroyed (daos_obj_close) + + } + + /// @note: performed RPCs: + /// - close catalogue kv (daos_obj_close) + + } + + current_ = indexes_[key]; + + return true; + +} + +void DaosCatalogueWriter::deselectIndex() { + + current_ = Index(); + currentIndexKey_ = Key(); + firstIndexWrite_ = false; + +} + +void DaosCatalogueWriter::clean() { + + flush(); + + deselectIndex(); + +} + +void DaosCatalogueWriter::close() { + + closeIndexes(); + +} + +const Index& DaosCatalogueWriter::currentIndex() { + + if (current_.null()) { + ASSERT(!currentIndexKey_.empty()); + selectIndex(currentIndexKey_); + } + + return current_; + +} + +/// @todo: other writers may be simultaneously updating the axes KeyValues in DAOS. Should these +/// new updates be retrieved and put into in-memory axes from time to time, e.g. every +/// time a value is put in an axis KeyValue? +void DaosCatalogueWriter::archive(const Key& key, std::unique_ptr fieldLocation) { + + if (current_.null()) { + ASSERT(!currentIndexKey_.empty()); + selectIndex(currentIndexKey_); + } + + /// @note: the current index timestamp is undefined at this point + Field field(std::move(fieldLocation), currentIndex().timestamp()); + + /// @todo: is sorting axes really necessary? + /// @note: sort in-memory axis values. Not triggering retrieval from DAOS axes. + const_cast(current_.axes()).sort(); + + /// before in-memory axes are updated as part of current_.put, we determine which + /// additions will need to be performed on axes in DAOS after the field gets indexed. + std::vector axesToExpand; + std::vector valuesToAdd; + std::string axisNames = ""; + std::string sep = ""; + + for (Key::const_iterator i = key.begin(); i != key.end(); ++i) { + + const std::string &keyword = i->first; + + std::string value = key.canonicalValue(keyword); + + if (value.length() == 0) continue; + + axisNames += sep + keyword; + sep = ","; + + /// @note: obtain in-memory axis values. Not triggering retrieval from DAOS axes. + /// @note: on first archive the in-memory axes will be empty and values() will return + /// empty sets. This is fine. + const auto& axis_set = current_.axes().values(keyword); + + //if (!axis_set.has_value() || !axis_set->get().contains(value)) { + if (!axis_set.contains(value)) { + + axesToExpand.push_back(keyword); + valuesToAdd.push_back(value); + + } + + } + + /// index the field and update in-memory axes + current_.put(key, field); + + fdb5::DaosSession s{}; + + /// persist axis names + if (firstIndexWrite_) { + + /// @todo: take oclass from config + fdb5::DaosKeyValueOID oid{currentIndexKey_.valuesToString(), OC_S1}; + fdb5::DaosKeyValueName n{pool_, db_cont_, oid}; + + /// @note: performed RPCs: + /// - generate index kv oid (daos_obj_generate_oid) + /// - ensure index kv exists (daos_obj_open) + fdb5::DaosKeyValue kv{s, n}; + + int axis_names_max_len = 512; + if (axisNames.length() > axis_names_max_len) + throw eckit::Exception("Serialised axis names exceeded configured maximum axis names length."); + + /// @note: performed RPCs: + /// - record axis names into index kv (daos_kv_put) + /// - close index kv when destroyed (daos_obj_close) + kv.put("axes", axisNames.data(), axisNames.length()); + + firstIndexWrite_ = false; + + } + + /// @todo: axes are supposed to be sorted before persisting. How do we do this with the DAOS approach? + /// sort axes every time they are loaded in the read pathway? + + if (axesToExpand.empty()) return; + + /// expand axis info in DAOS + while (!axesToExpand.empty()) { + + /// @todo: take oclass from config + fdb5::DaosKeyValueOID oid{currentIndexKey_.valuesToString() + std::string{"."} + axesToExpand.back(), OC_S1}; + fdb5::DaosKeyValueName n{pool_, db_cont_, oid}; + + /// @note: performed RPCs: + /// - generate axis kv oid (daos_obj_generate_oid) + /// - ensure axis kv exists (daos_obj_open) + fdb5::DaosKeyValue kv{s, n}; + + std::string v{"1"}; + + /// @note: performed RPCs: + /// - record axis value into axis kv (daos_kv_put) + /// - close axis kv when destroyed (daos_obj_close) + kv.put(valuesToAdd.back(), v.data(), v.length()); + + axesToExpand.pop_back(); + valuesToAdd.pop_back(); + + } + +} + +void DaosCatalogueWriter::flush() { + + if (!current_.null()) current_ = Index(); + +} + +void DaosCatalogueWriter::closeIndexes() { + + indexes_.clear(); // all indexes instances destroyed + +} + +static fdb5::CatalogueBuilder builder("daos.writer"); + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosCatalogueWriter.h b/src/fdb5/daos/DaosCatalogueWriter.h new file mode 100644 index 000000000..b738a6420 --- /dev/null +++ b/src/fdb5/daos/DaosCatalogueWriter.h @@ -0,0 +1,83 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Feb 2023 + +#pragma once + +#include "fdb5/daos/DaosCatalogue.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// DB that implements the FDB on DAOS + +class DaosCatalogueWriter : public DaosCatalogue, public CatalogueWriter { + +public: // methods + + DaosCatalogueWriter(const Key &key, const fdb5::Config& config); + DaosCatalogueWriter(const eckit::URI& uri, const fdb5::Config& config); + + virtual ~DaosCatalogueWriter() override; + + void index(const Key &key, const eckit::URI &uri, eckit::Offset offset, eckit::Length length) override { NOTIMP; }; + + void reconsolidate() override { NOTIMP; } + + /// Mount an existing TocCatalogue, which has a different metadata key (within + /// constraints) to allow on-line rebadging of data + /// variableKeys: The keys that are allowed to differ between the two DBs + void overlayDB(const Catalogue& otherCatalogue, const std::set& variableKeys, bool unmount) override { NOTIMP; }; + +// // Hide the contents of the DB!!! +// void hideContents() override; + +// bool enabled(const ControlIdentifier& controlIdentifier) const override; + + const Index& currentIndex() override; + +protected: // methods + + virtual bool selectIndex(const Key &key) override; + virtual void deselectIndex() override; + + bool open() override { NOTIMP; } + void flush() override; + void clean() override; + void close() override; + + void archive(const Key& key, std::unique_ptr fieldLocation) override; + + virtual void print( std::ostream &out ) const override { NOTIMP; } + +private: // methods + + void closeIndexes(); + +private: // types + + typedef std::map< Key, Index> IndexStore; + +private: // members + + IndexStore indexes_; + + Index current_; + + bool firstIndexWrite_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosCommon.cc b/src/fdb5/daos/DaosCommon.cc new file mode 100644 index 000000000..b7fcadb09 --- /dev/null +++ b/src/fdb5/daos/DaosCommon.cc @@ -0,0 +1,81 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "fdb5/daos/DaosCommon.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/config/Resource.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosName.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosCommon::DaosCommon(const fdb5::Config& config, const std::string& component, const fdb5::Key& key) { + + std::vector valid{"catalogue", "store"}; + ASSERT(std::find(valid.begin(), valid.end(), component) != valid.end()); + + db_cont_ = key.valuesToString(); + + readConfig(config, component, true); + +} + +DaosCommon::DaosCommon(const fdb5::Config& config, const std::string& component, const eckit::URI& uri) { + + /// @note: validity of input URI is not checked here because this constructor is only triggered + /// by DB::buildReader in EntryVisitMechanism, where validity of URIs is ensured beforehand + + fdb5::DaosName db_name{uri}; + pool_ = db_name.poolName(); + db_cont_ = db_name.containerName(); + + readConfig(config, component, false); + +} + +void DaosCommon::readConfig(const fdb5::Config& config, const std::string& component, bool readPool) { + + if (readPool) pool_ = "default"; + root_cont_ = "root"; + + eckit::LocalConfiguration c{}; + + if (config.has("daos")) c = config.getSubConfiguration("daos"); + if (readPool) + if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); + if (c.has(component)) root_cont_ = c.getSubConfiguration(component).getString("root_cont", root_cont_); + + std::string first_cap{component}; + first_cap[0] = toupper(component[0]); + + std::string all_caps{component}; + for (auto & c: all_caps) c = toupper(c); + + if (readPool) + pool_ = eckit::Resource("fdbDaos" + first_cap + "Pool;$FDB_DAOS_" + all_caps + "_POOL", pool_); + root_cont_ = eckit::Resource("fdbDaos" + first_cap + "RootCont;$FDB_DAOS_" + all_caps + "_ROOT_CONT", root_cont_); + + root_kv_.emplace(fdb5::DaosKeyValueName{pool_, root_cont_, main_kv_}); + db_kv_.emplace(fdb5::DaosKeyValueName{pool_, db_cont_, catalogue_kv_}); + + if (c.has("client")) + fdb5::DaosManager::instance().configure(c.getSubConfiguration("client")); + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosCommon.h b/src/fdb5/daos/DaosCommon.h new file mode 100644 index 000000000..5b077d5fc --- /dev/null +++ b/src/fdb5/daos/DaosCommon.h @@ -0,0 +1,57 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file DaosCommon.h +/// @author Nicolau Manubens +/// @date June 2023 + +#pragma once + +#include "eckit/filesystem/URI.h" + +#include "fdb5/database/Key.h" +#include "fdb5/config/Config.h" + +#include "fdb5/daos/DaosOID.h" +#include "fdb5/daos/DaosName.h" + +namespace fdb5 { + +class DaosCommon { + +public: // methods + + DaosCommon(const fdb5::Config&, const std::string& component, const fdb5::Key&); + DaosCommon(const fdb5::Config&, const std::string& component, const eckit::URI&); + + const fdb5::DaosKeyValueName& rootKeyValue() const { return root_kv_.value(); } + const fdb5::DaosKeyValueName& dbKeyValue() const { return db_kv_.value(); } + +private: // methods + + void readConfig(const fdb5::Config&, const std::string& component, bool readPool); + +protected: // members + + std::string component_; + + std::string pool_; + std::string root_cont_; + std::string db_cont_; + + eckit::Optional root_kv_; + eckit::Optional db_kv_; + + fdb5::DaosOID main_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config + fdb5::DaosOID catalogue_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config + +}; + +} \ No newline at end of file diff --git a/src/fdb5/daos/DaosContainer.cc b/src/fdb5/daos/DaosContainer.cc new file mode 100644 index 000000000..405a02794 --- /dev/null +++ b/src/fdb5/daos/DaosContainer.cc @@ -0,0 +1,270 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/TmpDir.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosOID.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosContainer::DaosContainer(DaosContainer&& other) noexcept : + pool_(other.pool_), + label_(std::move(other.label_)), coh_(std::move(other.coh_)), open_(other.open_), + oidAlloc_(std::move(other.oidAlloc_)) { + + other.open_ = false; + +} + +/// @todo: should forbid labels with UUID format? +DaosContainer::DaosContainer(fdb5::DaosPool& pool, const std::string& label) : pool_(pool), label_(label), open_(false) {} + +DaosContainer::~DaosContainer() { + + /// @todo: in destroy() and close() wed want to close and destroy/invalidate all object instances for + /// objects in the container. + /// What happens if we do obj.open() and then cont.close()? + + if (open_) close(); + +} + +void DaosContainer::create() { + + /// @todo: not sure what to do here. Should probably keep track of whether + // the container has been created or not via this DaosContainer instance, + // and return accordingly. But the container may have been destroyed by another + // process or DaosContainer instance. + + if (open_) return; + + ASSERT(label_.size() > 0); + + const daos_handle_t& poh = pool_.getOpenHandle(); + + try { + + DAOS_CALL(daos_cont_create_with_label(poh, label_.c_str(), NULL, NULL, NULL)); + + } catch (fdb5::DaosEntityAlreadyExistsException& e) {} + +} + +void DaosContainer::open() { + + if (open_) return; + + ASSERT(label_.size() > 0); + + const daos_handle_t& poh = pool_.getOpenHandle(); + + DAOS_CALL(daos_cont_open(poh, label_.c_str(), DAOS_COO_RW, &coh_, NULL, NULL)); + + open_ = true; + +} + +void DaosContainer::close() { + + if (!open_) { + eckit::Log::warning() << "Closing DaosContainer " << name() << ", container is not open" << std::endl; + return; + } + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => daos_cont_close()" << std::endl; + + int code = daos_cont_close(coh_, NULL); + + if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_cont_close(), file " + << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" + << code << ")" << std::endl; + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_cont_close()" << std::endl; + + open_ = false; + +} + +uint64_t DaosContainer::allocateOIDLo() { + + open(); + + if (oidAlloc_.num_oids == 0) { + oidAlloc_.num_oids = fdb5::DaosSession().containerOidsPerAlloc(); + DAOS_CALL(daos_cont_alloc_oids(coh_, oidAlloc_.num_oids + 1, &(oidAlloc_.next_oid), NULL)); + } else { + ++oidAlloc_.next_oid; + --oidAlloc_.num_oids; + } + + return oidAlloc_.next_oid; + +} + +fdb5::DaosArray DaosContainer::createArray(const daos_oclass_id_t& oclass, bool with_attr) { + + daos_otype_t otype = DAOS_OT_ARRAY; + + if (!with_attr) otype = DAOS_OT_ARRAY_BYTE; + + fdb5::DaosOID new_oid{0, allocateOIDLo(), otype, oclass}; + new_oid.generateReservedBits(*this); + + open(); + + fdb5::DaosArray obj(*this, new_oid, false); + obj.create(); + return obj; + +} + +fdb5::DaosArray DaosContainer::createArray(const fdb5::DaosOID& oid) { + + ASSERT(oid.otype() == DAOS_OT_ARRAY || oid.otype() == DAOS_OT_ARRAY_BYTE); + + open(); + + fdb5::DaosArray obj(*this, oid, false); + obj.create(); + return obj; + +} + +fdb5::DaosKeyValue DaosContainer::createKeyValue(const daos_oclass_id_t& oclass) { + + fdb5::DaosOID new_oid{0, allocateOIDLo(), DAOS_OT_KV_HASHED, oclass}; + new_oid.generateReservedBits(*this); + + open(); + + fdb5::DaosKeyValue obj(*this, new_oid, false); + obj.create(); + return obj; + +} + +fdb5::DaosKeyValue DaosContainer::createKeyValue(const fdb5::DaosOID& oid) { + + ASSERT(oid.otype() == DAOS_OT_KV_HASHED); + + open(); + + fdb5::DaosKeyValue obj(*this, oid, false); + obj.create(); + return obj; + +} + +std::vector DaosContainer::listOIDs() { + + /// @todo: proper memory management + /// @todo: auto snap destroyer + daos_epoch_t e; + DAOS_CALL( + daos_cont_create_snap_opt( + coh_, &e, NULL, (enum daos_snapshot_opts)(DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT), NULL + ) + ); + + daos_handle_t oith; + DAOS_CALL(daos_oit_open(coh_, e, &oith, NULL)); + + std::vector oids; + daos_anchor_t anchor = DAOS_ANCHOR_INIT; + int max_oids_per_rpc = 10; /// @todo: take from config + daos_obj_id_t oid_batch[max_oids_per_rpc]; + while (!daos_anchor_is_eof(&anchor)) { + uint32_t oids_nr = max_oids_per_rpc; + DAOS_CALL(daos_oit_list(oith, oid_batch, &oids_nr, &anchor, NULL)); + for (int i = 0; i < oids_nr; i++) { + oids.push_back(fdb5::DaosOID{oid_batch[i].hi, oid_batch[i].lo}); + } + } + + DAOS_CALL(daos_oit_close(oith, NULL)); + + daos_epoch_range_t epr{e, e}; + DAOS_CALL(daos_cont_destroy_snap(coh_, epr, NULL)); + + return oids; + +} + +const daos_handle_t& DaosContainer::getOpenHandle() { + + open(); + return coh_; + +}; + +bool DaosContainer::exists() { + + try { + + open(); + + } catch ( fdb5::DaosEntityNotFoundException& e) { + + return false; + + } + + return true; + +} + +std::string DaosContainer::name() const { + + return label_; + +} + +std::string DaosContainer::label() const { + + return label_; + +} + +fdb5::DaosPool& DaosContainer::getPool() const { + + return pool_; + +} + +AutoContainerDestroy::~AutoContainerDestroy() noexcept(false) { + + bool fail = !eckit::Exception::throwing(); + + try { + cont_.getPool().destroyContainer(cont_.label()); + } catch (std::exception& e) { + eckit::Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; + if (fail) { + eckit::Log::error() << "** Exception is re-thrown" << std::endl; + throw; + } + eckit::Log::error() << "** An exception is already in progress" << std::endl; + eckit::Log::error() << "** Exception is ignored" << std::endl; + } + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosContainer.h b/src/fdb5/daos/DaosContainer.h new file mode 100644 index 000000000..2268a80fd --- /dev/null +++ b/src/fdb5/daos/DaosContainer.h @@ -0,0 +1,91 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jul 2022 + +#pragma once + +#include + +#include + +#include "fdb5/daos/DaosObject.h" + +struct OidAlloc { + uint64_t next_oid; + int num_oids; +}; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosPool; + +class DaosContainer { + +public: // methods + + DaosContainer(DaosContainer&&) noexcept; + ~DaosContainer(); + + uint64_t allocateOIDLo(); + fdb5::DaosArray createArray(const daos_oclass_id_t& oclass = OC_S1, bool with_attr = true); + fdb5::DaosArray createArray(const fdb5::DaosOID&); + fdb5::DaosKeyValue createKeyValue(const daos_oclass_id_t& oclass = OC_S1); + fdb5::DaosKeyValue createKeyValue(const fdb5::DaosOID&); + std::vector listOIDs(); + + const daos_handle_t& getOpenHandle(); + + std::string name() const; + std::string label() const; + fdb5::DaosPool& getPool() const; + +private: // methods + + friend class DaosPool; + + DaosContainer(fdb5::DaosPool&, const std::string&); + + void open(); + void close(); + void create(); + bool exists(); + +private: // members + + fdb5::DaosPool& pool_; + std::string label_ = std::string(); + daos_handle_t coh_; + bool open_; + + OidAlloc oidAlloc_{}; + +}; + +class AutoContainerDestroy { + +public: // methods + + AutoContainerDestroy(fdb5::DaosContainer& cont) : cont_(cont) {} + + ~AutoContainerDestroy() noexcept(false); + +private: // members + + fdb5::DaosContainer& cont_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosEngine.cc b/src/fdb5/daos/DaosEngine.cc new file mode 100644 index 000000000..7a12b98ee --- /dev/null +++ b/src/fdb5/daos/DaosEngine.cc @@ -0,0 +1,253 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/LibFdb5.h" +#include "fdb5/daos/DaosEngine.h" +#include "eckit/serialisation/MemoryStream.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosName.h" + +using namespace eckit; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +void DaosEngine::configureDaos(const Config& config) const { + + if (daos_config_.has_value()) return; + + daos_config_.emplace(eckit::LocalConfiguration()); + if (config.has("daos")) daos_config_.emplace(config.getSubConfiguration("daos")); + if (daos_config_->has("client")) + fdb5::DaosManager::instance().configure(daos_config_->getSubConfiguration("client")); + +} + +std::string DaosEngine::name() const { + return DaosEngine::typeName(); +} + +bool DaosEngine::canHandle(const eckit::URI& uri, const Config& config) const { + + configureDaos(config); + + if (uri.scheme() != "daos") + return false; + + fdb5::DaosName n{uri}; + + if (!n.hasOID()) return false; + + /// @todo: check containerName is not root_cont_. root_cont_ should be populated in + /// configureDaos as done in DaosCommon + // bool is_root_name = (n.containerName().find(root_cont_) != std::string::npos); + bool is_root_name = false; + bool is_store_name = (n.containerName().find("_") != std::string::npos); + + /// @note: performed RPCs: + /// - generate oids (daos_obj_generate_oid) + /// - db kv open (daos_kv_open) + + fdb5::DaosName n2{n.poolName(), n.containerName(), catalogue_kv_}; + bool is_catalogue_kv = (!is_root_name && !is_store_name && (n.OID() == n2.OID())); + + return is_catalogue_kv && n.exists(); + +} + +std::vector DaosEngine::visitableLocations(const Key& key, const Config& config) const +{ + + /// @note: code mostly copied from DaosCommon + /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is + /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily + /// initialised. If owning a private instance of DaosCommon here, then the private members of + /// DaosCommon are not accessible from here + + configureDaos(config); + + std::string pool = "default"; + std::string root_cont = "root"; + + if (daos_config_->has("catalogue")) pool = daos_config_->getSubConfiguration("catalogue").getString("pool", pool); + if (daos_config_->has("catalogue")) root_cont = daos_config_->getSubConfiguration("catalogue").getString("root_cont", root_cont); + + pool = eckit::Resource("fdbDaosCataloguePool;$FDB_DAOS_CATALOGUE_POOL", pool); + root_cont = eckit::Resource("fdbDaosCatalogueRootCont;$FDB_DAOS_CATALOGUE_ROOT_CONT", root_cont); + + fdb5::DaosOID main_kv_oid{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config + + /// --- + + fdb5::DaosKeyValueName main_kv_name{pool, root_cont, main_kv_oid}; + + fdb5::DaosSession s{}; + + std::vector res{}; + + /// @note: performed RPCs: + /// - main kv open (daos_kv_open) + + /// @todo: use this Optional technique in all cases where an action needs to be performed + /// (i.e. not just throw an exception) if an object does not exist + eckit::Optional main_kv; + try { + main_kv.emplace(s, main_kv_name); + } catch (fdb5::DaosEntityNotFoundException& e) { + return res; + } + + /// @note: performed RPCs: + /// - main kv list keys (daos_kv_list) + for (const auto& k : main_kv->keys()) { + + try { + + /// @note: performed RPCs: + /// - main kv get db location size (daos_kv_get without a buffer) + /// - main kv get db location (daos_kv_get) + uint64_t size = main_kv->size(k); + std::vector v(size); + main_kv->get(k, v.data(), size); + + eckit::URI uri(std::string(v.begin(), v.end())); + ASSERT(uri.scheme() == typeName()); + + /// @todo: this exact deserialisation is performed twice. Once here and once + /// in DaosCatalogue::(uri, ...). Try to avoid one. + fdb5::DaosKeyValueName db_kv_name{uri}; + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db key get size (daos_kv_get without a buffer) + /// - db key get (daos_kv_get) + fdb5::DaosKeyValue db_kv{s, db_kv_name}; /// @note: includes exist check + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + fdb5::Key db_key(ms); + + if (db_key.match(key)) { + + Log::debug() << " found match with " << main_kv_name.URI() << " at key " << k << std::endl; + res.push_back(uri); + + } + + } catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database " << k << " from " << main_kv_name.URI() << std::endl; + eckit::Log::error() << e.what() << std::endl; + } + + } + + return res; + +} + +std::vector DaosEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const +{ + + /// @todo: in the POSIX backend, a set of possible visitable locations is first constructed + /// from the schema, and the set of visitable locations found in the root directories are + /// checked against that set. In the DAOS backend the first step is skipped. + + /// @note: code mostly copied from DaosCommon + /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is + /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily + /// initialised. If owning a private instance of DaosCommon here, then the private members of + /// DaosCommon are not accessible from here + + configureDaos(config); + + std::string pool = "default"; + std::string root_cont = "root"; + + if (daos_config_->has("catalogue")) pool = daos_config_->getSubConfiguration("catalogue").getString("pool", pool); + if (daos_config_->has("catalogue")) root_cont = daos_config_->getSubConfiguration("catalogue").getString("root_cont", root_cont); + + pool = eckit::Resource("fdbDaosCataloguePool;$FDB_DAOS_CATALOGUE_POOL", pool); + root_cont = eckit::Resource("fdbDaosCatalogueRootCont;$FDB_DAOS_CATALOGUE_ROOT_CONT", root_cont); + + fdb5::DaosOID main_kv_oid{0, 0, DAOS_OT_KV_HASHED, OC_S1}; /// @todo: take oclass from config + + /// --- + + fdb5::DaosKeyValueName main_kv_name{pool, root_cont, main_kv_oid}; + + fdb5::DaosSession s{}; + + std::vector res{}; + + /// @note: performed RPCs: + /// - main kv open (daos_kv_open) + + /// @todo: use this Optional technique in all cases where an action needs to be performed + /// (i.e. not just throw an exception) if an object does not exist + eckit::Optional main_kv; + try { + main_kv.emplace(s, main_kv_name); + } catch (fdb5::DaosEntityNotFoundException& e) { + return res; + } + + /// @note: performed RPCs: + /// - main kv list keys (daos_kv_list) + for (const auto& k : main_kv->keys()) { + + try { + + /// @note: performed RPCs: + /// - main kv get db location size (daos_kv_get without a buffer) + /// - main kv get db location (daos_kv_get) + uint64_t size = main_kv->size(k); + std::vector v(size); + main_kv->get(k, v.data(), size); + + eckit::URI uri(std::string(v.begin(), v.end())); + ASSERT(uri.scheme() == typeName()); + + /// @todo: this exact deserialisation is performed twice. Once here and once + /// in DaosCatalogue::(uri, ...). Try to avoid one. + fdb5::DaosKeyValueName db_kv_name{uri}; + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db key get size (daos_kv_get without a buffer) + /// - db key get (daos_kv_get) + fdb5::DaosKeyValue db_kv{s, db_kv_name}; + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + fdb5::Key db_key(ms); + + if (db_key.partialMatch(request)) { + + Log::debug() << " found match with " << main_kv_name.URI() << " at key " << k << std::endl; + res.push_back(uri); + + } + + } catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database from " << main_kv_name.URI() << std::endl; + eckit::Log::error() << e.what() << std::endl; + } + + } + + return res; + +} + +static EngineBuilder daos_builder; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosEngine.h b/src/fdb5/daos/DaosEngine.h new file mode 100644 index 000000000..690f4d26f --- /dev/null +++ b/src/fdb5/daos/DaosEngine.h @@ -0,0 +1,67 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Feb 2022 + +#pragma once + +#include "eckit/utils/Optional.h" + +#include "fdb5/database/Engine.h" + +#include "fdb5/daos/DaosOID.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosEngine : public fdb5::Engine { + +public: // methods + + DaosEngine() {}; + + static const char* typeName() { return "daos"; } + +protected: // methods + + virtual std::string name() const override; + + virtual std::string dbType() const override { NOTIMP; }; + + virtual eckit::URI location(const Key &key, const Config& config) const override { NOTIMP; }; + + virtual bool canHandle(const eckit::URI&, const Config&) const override; + + virtual std::vector allLocations(const Key& key, const Config& config) const override { NOTIMP; }; + + virtual std::vector visitableLocations(const Key& key, const Config& config) const override; + virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; + + virtual std::vector writableLocations(const Key& key, const Config& config) const override { NOTIMP; }; + + virtual void print( std::ostream &out ) const override { NOTIMP; }; + +private: // methods + + void configureDaos(const Config&) const; + +private: // members + + mutable eckit::Optional daos_config_; + fdb5::DaosOID catalogue_kv_{0, 0, DAOS_OT_KV_HASHED, OC_S1}; // take oclass from config + +}; + +//---------------------------------------------------------------------------------------------------------------------- + + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosException.cc b/src/fdb5/daos/DaosException.cc new file mode 100644 index 000000000..9780cf0d0 --- /dev/null +++ b/src/fdb5/daos/DaosException.cc @@ -0,0 +1,34 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/daos/DaosException.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosException::DaosException(const std::string& w) : Exception(w) {} + +DaosException::DaosException(const std::string& w, const eckit::CodeLocation& l) : + Exception(w, l) {} + +DaosEntityNotFoundException::DaosEntityNotFoundException(const std::string& w) : DaosException(w) {} + +DaosEntityNotFoundException::DaosEntityNotFoundException(const std::string& w, const eckit::CodeLocation& l) : + DaosException(w, l) {} + +DaosEntityAlreadyExistsException::DaosEntityAlreadyExistsException(const std::string& w) : DaosException(w) {} + +DaosEntityAlreadyExistsException::DaosEntityAlreadyExistsException(const std::string& w, const eckit::CodeLocation& l) : + DaosException(w, l) {} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosException.h b/src/fdb5/daos/DaosException.h new file mode 100644 index 000000000..a2fb153fc --- /dev/null +++ b/src/fdb5/daos/DaosException.h @@ -0,0 +1,44 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Dec 2022 + +#pragma once + +#include "eckit/exception/Exceptions.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosException : public eckit::Exception { +public: + DaosException(const std::string&); + DaosException(const std::string&, const eckit::CodeLocation&); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosEntityNotFoundException : public DaosException { +public: + DaosEntityNotFoundException(const std::string&); + DaosEntityNotFoundException(const std::string&, const eckit::CodeLocation&); +}; + +class DaosEntityAlreadyExistsException : public DaosException { +public: + DaosEntityAlreadyExistsException(const std::string&); + DaosEntityAlreadyExistsException(const std::string&, const eckit::CodeLocation&); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosFieldLocation.cc b/src/fdb5/daos/DaosFieldLocation.cc new file mode 100644 index 000000000..f6a6ea4b8 --- /dev/null +++ b/src/fdb5/daos/DaosFieldLocation.cc @@ -0,0 +1,110 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/filesystem/URIManager.h" +#include "fdb5/daos/DaosFieldLocation.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/LibFdb5.h" + +namespace fdb5 { + +::eckit::ClassSpec DaosFieldLocation::classSpec_ = {&FieldLocation::classSpec(), "DaosFieldLocation",}; +::eckit::Reanimator DaosFieldLocation::reanimator_; + +//---------------------------------------------------------------------------------------------------------------------- + +DaosFieldLocation::DaosFieldLocation(const DaosFieldLocation& rhs) : + FieldLocation(rhs.uri_, rhs.offset_, rhs.length_, rhs.remapKey_) {} + +DaosFieldLocation::DaosFieldLocation(const eckit::URI &uri) : FieldLocation(uri) {} + +/// @todo: remove remapKey from signature and always pass empty Key to FieldLocation +DaosFieldLocation::DaosFieldLocation(const eckit::URI &uri, eckit::Offset offset, eckit::Length length, const Key& remapKey) : + FieldLocation(uri, offset, length, remapKey) {} + +DaosFieldLocation::DaosFieldLocation(eckit::Stream& s) : + FieldLocation(s) {} + +std::shared_ptr DaosFieldLocation::make_shared() const { + return std::make_shared(std::move(*this)); +} + +eckit::DataHandle* DaosFieldLocation::dataHandle() const { + + return fdb5::DaosArrayName(uri_).dataHandle(offset(), length()); + +} + +void DaosFieldLocation::print(std::ostream &out) const { + out << "DaosFieldLocation[uri=" << uri_ << "]"; +} + +void DaosFieldLocation::visit(FieldLocationVisitor& visitor) const { + visitor(*this); +} + +static FieldLocationBuilder builder("daos"); + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosURIManager : public eckit::URIManager { + virtual bool query() override { return true; } + virtual bool fragment() override { return true; } + + virtual eckit::PathName path(const eckit::URI& f) const override { return f.name(); } + + virtual bool exists(const eckit::URI& f) override { + + return fdb5::DaosName(f).exists(); + + } + + virtual eckit::DataHandle* newWriteHandle(const eckit::URI& f) override { + + if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; + + return fdb5::DaosArrayName(f).dataHandle(); + + } + + virtual eckit::DataHandle* newReadHandle(const eckit::URI& f) override { + + if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; + + return fdb5::DaosArrayName(f).dataHandle(); + + } + + virtual eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, const eckit::LengthList& ll) override { + + if (fdb5::DaosName(f).OID().otype() != DAOS_OT_ARRAY) NOTIMP; + + return fdb5::DaosArrayName(f).dataHandle(); + + } + + virtual std::string asString(const eckit::URI& uri) const override { + std::string q = uri.query(); + if (!q.empty()) + q = "?" + q; + std::string f = uri.fragment(); + if (!f.empty()) + f = "#" + f; + + return uri.scheme() + ":" + uri.name() + q + f; + } +public: + DaosURIManager(const std::string& name) : eckit::URIManager(name) {} +}; + +static DaosURIManager daos_uri_manager("daos"); + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosFieldLocation.h b/src/fdb5/daos/DaosFieldLocation.h new file mode 100644 index 000000000..13abeba45 --- /dev/null +++ b/src/fdb5/daos/DaosFieldLocation.h @@ -0,0 +1,61 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Nov 2022 + +#pragma once + +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" + +#include "fdb5/database/FieldLocation.h" + +#include "fdb5/daos/DaosName.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosFieldLocation : public FieldLocation { +public: + + DaosFieldLocation(const DaosFieldLocation& rhs); + DaosFieldLocation(const eckit::URI &uri); + DaosFieldLocation(const eckit::URI& uri, eckit::Offset offset, eckit::Length length, const Key& remapKey); + DaosFieldLocation(eckit::Stream&); + + eckit::DataHandle* dataHandle() const override; + + virtual std::shared_ptr make_shared() const override; + + virtual void visit(FieldLocationVisitor& visitor) const override; + +public: // For Streamable + + static const eckit::ClassSpec& classSpec() { return classSpec_;} + +protected: // For Streamable + + virtual const eckit::ReanimatorBase& reanimator() const override { return reanimator_; } + + static eckit::ClassSpec classSpec_; + static eckit::Reanimator reanimator_; + +private: // methods + + void print(std::ostream &out) const override; + +}; + + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosIndex.cc b/src/fdb5/daos/DaosIndex.cc new file mode 100644 index 000000000..c3225845c --- /dev/null +++ b/src/fdb5/daos/DaosIndex.cc @@ -0,0 +1,271 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include // for PATH_MAX + +#include "eckit/io/MemoryHandle.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/HandleStream.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosIndex.h" +#include "fdb5/daos/DaosLazyFieldLocation.h" + +fdb5::DaosKeyValueName buildIndexKvName(const fdb5::Key& key, const fdb5::DaosName& name) { + + ASSERT(!name.hasOID()); + + /// create index kv + /// @todo: pass oclass from config + /// @todo: hash string into lower oid bits + fdb5::DaosKeyValueOID index_kv_oid{key.valuesToString(), OC_S1}; + + return fdb5::DaosKeyValueName{name.poolName(), name.containerName(), index_kv_oid}; + +} + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosIndex::DaosIndex(const Key& key, const fdb5::DaosName& name) : + IndexBase(key, "daosKeyValue"), + location_(buildIndexKvName(key, name), 0) { + + fdb5::DaosSession s{}; + + /// @note: performed RPCs: + /// - generate index kv oid (daos_obj_generate_oid) + /// - create/open index kv (daos_kv_open) + fdb5::DaosKeyValue index_kv_obj{s, location_.daosName()}; + + /// write indexKey under "key" + eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + hs << key; + } + + int idx_key_max_len = 512; + + if (hs.bytesWritten() > idx_key_max_len) + throw eckit::Exception("Serialised index key exceeded configured maximum index key length."); + + /// @note: performed RPCs: + /// - record index key into index kv (daos_kv_put) + index_kv_obj.put("key", h.data(), hs.bytesWritten()); + +} + +DaosIndex::DaosIndex(const Key& key, const fdb5::DaosKeyValueName& name, bool readAxes) : + IndexBase(key, "daosKeyValue"), + location_(name, 0) { + + if (readAxes) updateAxes(); + +} + +void DaosIndex::updateAxes() { + + fdb5::DaosSession s{}; + const fdb5::DaosKeyValueName& index_kv_name = location_.daosName(); + + /// @note: performed RPCs: + /// - ensure axis kv exists (daos_obj_open) + fdb5::DaosKeyValue index_kv{s, index_kv_name}; + + int axis_names_max_len = 512; /// @todo: take from config + std::vector axes_data((long) axis_names_max_len); + + /// @note: performed RPCs: + /// - get axes key size and content (daos_kv_get without buffer + daos_kv_get) + long res = index_kv.get("axes", &axes_data[0], axis_names_max_len); + + std::vector axis_names; + eckit::Tokenizer parse(","); + parse(std::string(axes_data.begin(), std::next(axes_data.begin(), res)), axis_names); + std::string indexKey{key_.valuesToString()}; + for (const auto& name : axis_names) { + /// @todo: take oclass from config + fdb5::DaosKeyValueOID oid(indexKey + std::string{"."} + name, OC_S1); + fdb5::DaosKeyValueName nkv(index_kv_name.poolName(), index_kv_name.containerName(), oid); + + /// @note: performed RPCs: + /// - generate axis kv oid (daos_obj_generate_oid) + /// - ensure axis kv exists (daos_obj_open) + fdb5::DaosKeyValue axis_kv{s, nkv}; + + /// @note: performed RPCs: + /// - one or more kv list (daos_kv_list) + axes_.insert(name, axis_kv.keys()); + } + + axes_.sort(); + +} + +bool DaosIndex::get(const Key &key, const Key &remapKey, Field &field) const { + + const fdb5::DaosKeyValueName& n = location_.daosName(); + + fdb5::DaosSession s{}; + + /// @note: performed RPCs: + /// - ensure index kv exists (daos_obj_open) + fdb5::DaosKeyValue index{s, n}; + + std::string query{key.valuesToString()}; + + int field_loc_max_len = 512; /// @todo: read from config + std::vector loc_data((long) field_loc_max_len); + long res; + + try { + + /// @note: performed RPCs: + /// - retrieve field array location from index kv (daos_kv_get) + res = index.get(query, &loc_data[0], field_loc_max_len); + + } catch (fdb5::DaosEntityNotFoundException& e) { + + /// @note: performed RPCs: + /// - close index kv (daos_obj_close) + + return false; + + } + + eckit::MemoryStream ms{&loc_data[0], (size_t) res}; + + /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. + time_t ts; + ms >> ts; + + fdb5::FieldLocation* loc = eckit::Reanimator::reanimate(ms); + field = fdb5::Field(std::move(*loc), ts, fdb5::FieldDetails()); + + /// @note: performed RPCs: + /// - close index kv (daos_obj_close) + + return true; + +} + +void DaosIndex::add(const Key &key, const Field &field) { + + eckit::MemoryHandle h{(size_t) PATH_MAX}; + eckit::HandleStream hs{h}; + h.openForWrite(eckit::Length(0)); + { + eckit::AutoClose closer(h); + /// @note: in the POSIX back-end, keeping a timestamp per index is necessary, to allow + /// determining which was the latest indexed field in cases where multiple processes + /// index a same field or in cases where multiple catalogues are combined with DistFDB. + /// In the DAOS back-end, however, determining the latest indexed field is straigthforward + /// as all parallel processes writing fields for a same index key will share a DAOS + /// key-value, and the last indexing will supersede the previous ones. + /// DistFDB will be obsoleted in favour of a centralised catalogue mechanism which can + /// index fields on multiple catalogues. + /// Therefore keeping timestamps in DAOS should not be necessary. + /// They are kept for now only for informational purposes. + takeTimestamp(); + hs << timestamp(); + hs << field.location(); + } + + int field_loc_max_len = 512; /// @todo: read from config + if (hs.bytesWritten() > field_loc_max_len) + throw eckit::Exception("Serialised field location exceeded configured maximum location length."); + + fdb5::DaosSession s{}; + + /// @note: performed RPCs: + /// - ensure index kv exists (daos_obj_open) + /// - record field key and location into index kv (daos_kv_put) + /// - close index kv when destroyed (daos_obj_close) + fdb5::DaosKeyValue{s, location_.daosName()}.put(key.valuesToString(), h.data(), hs.bytesWritten()); + +} + +void DaosIndex::entries(EntryVisitor &visitor) const { + + Index instantIndex(const_cast(this)); + + // Allow the visitor to selectively decline to visit the entries in this index + if (visitor.visitIndex(instantIndex)) { + + fdb5::DaosSession s{}; + + /// @note: performed RPCs: + /// - index kv open (daos_obj_open) + /// - index kv list keys (daos_kv_list) + fdb5::DaosKeyValue index_kv{s, location_.daosName()}; + + for (const auto& key : index_kv.keys()) { + + if (key == "axes" || key == "key") continue; + + /// @note: the DaosCatalogue is currently indexing a serialised DaosFieldLocation for each + /// archived field key. In the list pathway, DaosLazyFieldLocations are built for all field + /// keys present in an index -- without retrieving the actual location --, and + /// ListVisitor::visitDatum is called for each (see note at the top of DaosLazyFieldLocation.h). + /// When a field key is matched in visitDatum, DaosLazyFieldLocation::stableLocation is called, + /// which in turn calls this method here and triggers retrieval and deserialisation of the + /// indexed DaosFieldLocation, and returns it. Since the deserialised instance is of a + /// polymorphic class, it needs to be reanimated. + fdb5::FieldLocation* loc = new fdb5::DaosLazyFieldLocation(location_.daosName(), key); + fdb5::Field field(std::move(*loc), time_t(), fdb5::FieldDetails()); + visitor.visitDatum(field, key); + + } + + } +} + +const std::vector DaosIndex::dataURIs() const { + + /// @note: if daos index + daos store, this will return a uri to a DAOS array for each indexed field + /// @note: if daos index + posix store, this will return a vector of unique uris to all referenced posix files + /// in this index (one for each writer process that has written to the index) + /// @note: in the case where we have a daos store, the current implementation of dataURIs is unnecessarily inefficient. + /// This method is only called in DaosWipeVisitor, where the uris obtained from this method are processed to obtain + /// unique store container paths - will always result in just one container uri! Having a URI store for each index in + /// DAOS could make this process more efficient, but it would imply more KV operations and slow down field writes. + /// @note: in the case where we have a posix store there will be more than one unique store file paths. The current + /// implementation is still inefficient but preferred to maintaining a URI store in the DAOS catalogue + + fdb5::DaosSession s{}; + fdb5::DaosKeyValue index_kv{s, location_.daosName()}; + + std::set res; + + for (const auto& key : index_kv.keys()) { + + if (key == "axes" || key == "key") continue; + + std::vector data; + eckit::MemoryStream ms = index_kv.getMemoryStream(data, key, "index kv"); + + time_t ts; + ms >> ts; + + std::unique_ptr fl(eckit::Reanimator::reanimate(ms)); + res.insert(fl->uri()); + + } + + return std::vector(res.begin(), res.end()); + +} + +//----------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosIndex.h b/src/fdb5/daos/DaosIndex.h new file mode 100644 index 000000000..81df96102 --- /dev/null +++ b/src/fdb5/daos/DaosIndex.h @@ -0,0 +1,72 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Mar 2023 + +#pragma once + +#include "fdb5/database/Index.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosIndexLocation.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + + +class DaosIndex : public IndexBase { + +public: // methods + + /// @note: creates a new index in DAOS, in the container pointed to by 'name' + DaosIndex(const Key& key, const fdb5::DaosName& name); + /// @note: used to represent and operate with an index which already exists in DAOS + DaosIndex(const Key& key, const fdb5::DaosKeyValueName& name, bool readAxes = true); + + void flock() const override { NOTIMP; } + void funlock() const override { NOTIMP; } + +private: // methods + + const IndexLocation& location() const override { return location_; } + const std::vector dataURIs() const override; + + bool dirty() const override { NOTIMP; } + + void open() override { NOTIMP; }; + void close() override { NOTIMP; } + void reopen() override { NOTIMP; } + + void visit(IndexLocationVisitor& visitor) const override { NOTIMP; } + + bool get( const Key &key, const Key &remapKey, Field &field ) const override; + void add( const Key &key, const Field &field ) override; + void flush() override { NOTIMP; } + void encode(eckit::Stream& s, const int version) const override { NOTIMP; } + void entries(EntryVisitor& visitor) const override; + + void print( std::ostream &out ) const override { NOTIMP; } + void dump(std::ostream& out, const char* indent, bool simple = false, bool dumpFields = false) const override { NOTIMP; } + + IndexStats statistics() const override { NOTIMP; } + + /// @note: reads complete axis info from DAOS. + void updateAxes(); + +private: // members + + fdb5::DaosIndexLocation location_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosIndexLocation.cc b/src/fdb5/daos/DaosIndexLocation.cc new file mode 100644 index 000000000..8d2508332 --- /dev/null +++ b/src/fdb5/daos/DaosIndexLocation.cc @@ -0,0 +1,27 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/daos/DaosIndexLocation.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosIndexLocation::DaosIndexLocation(const fdb5::DaosKeyValueName& name, off_t offset) : name_(name), offset_(offset) {} + +void DaosIndexLocation::print(std::ostream &out) const { + + out << "(" << name_.URI().asString() << ":" << offset_ << ")"; + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosIndexLocation.h b/src/fdb5/daos/DaosIndexLocation.h new file mode 100644 index 000000000..217dfb6d8 --- /dev/null +++ b/src/fdb5/daos/DaosIndexLocation.h @@ -0,0 +1,56 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Mar 2023 + +#pragma once + +#include "eckit/exception/Exceptions.h" + +#include "fdb5/database/IndexLocation.h" + +#include "fdb5/daos/DaosName.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosIndexLocation : public IndexLocation { + +public: // methods + + DaosIndexLocation(const fdb5::DaosKeyValueName& name, off_t offset); + + eckit::URI uri() const override { return name_.URI(); } + + IndexLocation* clone() const override { NOTIMP; } + + const fdb5::DaosKeyValueName& daosName() const { return name_; }; + +protected: // For Streamable + + void encode(eckit::Stream&) const override { NOTIMP; } + +private: // methods + + void print(std::ostream &out) const override; + +private: // members + + fdb5::DaosKeyValueName name_; + + off_t offset_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosKeyValueHandle.cc b/src/fdb5/daos/DaosKeyValueHandle.cc new file mode 100644 index 000000000..bd3fa17cb --- /dev/null +++ b/src/fdb5/daos/DaosKeyValueHandle.cc @@ -0,0 +1,161 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "eckit/exception/Exceptions.h" + +#include "fdb5/daos/DaosKeyValueHandle.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosException.h" + +using eckit::Length; +using eckit::Offset; + +namespace fdb5 { + +DaosKeyValueHandle::DaosKeyValueHandle(const fdb5::DaosKeyValueName& name, const std::string& key) : name_(name), key_(key), open_(false), offset_(0) {} + +DaosKeyValueHandle::~DaosKeyValueHandle() { + + if (open_) eckit::Log::error() << "DaosKeyValueHandle not closed before destruction." << std::endl; + +} + +void DaosKeyValueHandle::print(std::ostream& s) const { + s << "DaosKeyValueHandle[notimp]"; +} + +void DaosKeyValueHandle::openForWrite(const Length& len) { + + if (open_) throw eckit::SeriousBug{"Handle already opened."}; + + session(); + + /// @todo: alternatively call name_.create() and the like + fdb5::DaosPool& p = session_->getPool(name_.poolName()); + fdb5::DaosContainer& c = p.ensureContainer(name_.containerName()); + + /// @note: only way to check kv existence without generating a snapshot is + /// to attempt open, which results in creation without an error rc if n.e. + /// A kv open is performed in both c.createKeyValue and DaosKeyValue(session, name). + /// The former is used here. + kv_.emplace( c.createKeyValue(name_.OID()) ); + + kv_->open(); + + open_ = true; + + offset_ = eckit::Offset(0); + +} + +Length DaosKeyValueHandle::openForRead() { + + if (open_) throw eckit::SeriousBug{"Handle already opened."}; + + session(); + + kv_.emplace(session_.value(), name_); + + kv_->open(); + + open_ = true; + + offset_ = eckit::Offset(0); + + return Length(kv_->size(key_)); + +} + +long DaosKeyValueHandle::write(const void* buf, long len) { + + ASSERT(open_); + + long written = kv_->put(key_, buf, len); + + offset_ += written; + + return written; + +} + +long DaosKeyValueHandle::read(void* buf, long len) { + + ASSERT(open_); + + long read = kv_->get(key_, buf, len); + + offset_ += read; + + return read; + +} + +void DaosKeyValueHandle::close() { + + if (!open_) return; + + kv_->close(); + + open_ = false; + +} + +void DaosKeyValueHandle::flush() { + + /// empty implmenetation + +} + +Length DaosKeyValueHandle::size() { + + fdb5::DaosKeyValue kv{session(), name_}; + + return Length(kv.size(key_)); + +} + +Length DaosKeyValueHandle::estimate() { + + return size(); + +} + +Offset DaosKeyValueHandle::position() { + + /// @todo: should position() crash if unopened? + return offset_; + +} + +bool DaosKeyValueHandle::canSeek() const { + + return false; + +} + +std::string DaosKeyValueHandle::title() const { + + return name_.asString(); + +} + +fdb5::DaosSession& DaosKeyValueHandle::session() { + + if (!session_.has_value()) session_.emplace(); + return session_.value(); + +} + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosKeyValueHandle.h b/src/fdb5/daos/DaosKeyValueHandle.h new file mode 100644 index 000000000..d0193b94d --- /dev/null +++ b/src/fdb5/daos/DaosKeyValueHandle.h @@ -0,0 +1,71 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Feb 2023 + +#pragma once + +#include "eckit/io/DataHandle.h" + +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosKeyValue; + +class DaosKeyValueName; + +class DaosKeyValueHandle : public eckit::DataHandle { + +public: // methods + + DaosKeyValueHandle(const fdb5::DaosKeyValueName&, const std::string& key); + + ~DaosKeyValueHandle(); + + virtual void print(std::ostream&) const override; + + virtual void openForWrite(const eckit::Length&) override; + virtual eckit::Length openForRead() override; + + virtual long write(const void*, long) override; + virtual long read(void*, long) override; + virtual void close() override; + virtual void flush() override; + + virtual eckit::Length size() override; + virtual eckit::Length estimate() override; + virtual eckit::Offset position() override; + virtual bool canSeek() const override; + + virtual std::string title() const override; + +private: // methods + + fdb5::DaosSession& session(); + +private: // members + + mutable fdb5::DaosKeyValueName name_; + std::string key_; + std::optional session_; + std::optional kv_; + bool open_; + eckit::Offset offset_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosLazyFieldLocation.cc b/src/fdb5/daos/DaosLazyFieldLocation.cc new file mode 100644 index 000000000..b557ef9f8 --- /dev/null +++ b/src/fdb5/daos/DaosLazyFieldLocation.cc @@ -0,0 +1,70 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/serialisation/MemoryStream.h" + +#include "fdb5/daos/DaosLazyFieldLocation.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosLazyFieldLocation::DaosLazyFieldLocation(const fdb5::DaosLazyFieldLocation& rhs) : + FieldLocation(), index_(rhs.index_), key_(rhs.key_) {} + +DaosLazyFieldLocation::DaosLazyFieldLocation(const fdb5::DaosKeyValueName& index, const std::string& key) : + FieldLocation(), index_(index), key_(key) {} + +std::shared_ptr DaosLazyFieldLocation::make_shared() const { + return std::make_shared(std::move(*this)); +} + +eckit::DataHandle* DaosLazyFieldLocation::dataHandle() const { + + return realise()->dataHandle(); + +} + +void DaosLazyFieldLocation::print(std::ostream &out) const { + out << *realise(); +} + +void DaosLazyFieldLocation::visit(FieldLocationVisitor& visitor) const { + realise()->visit(visitor); +} + +std::shared_ptr DaosLazyFieldLocation::stableLocation() const { + return realise()->make_shared(); +} + +std::unique_ptr& DaosLazyFieldLocation::realise() const { + + if (fl_) return fl_; + + /// @note: performed RPCs: + /// - index kv get (daos_kv_get) + fdb5::DaosSession s{}; + fdb5::DaosKeyValue index_kv{s, index_}; + std::vector data; + eckit::MemoryStream ms = index_kv.getMemoryStream(data, key_, "index kv"); + + /// @note: timestamp read for informational purpoes. See note in DaosIndex::add. + time_t ts; + ms >> ts; + + fl_.reset(eckit::Reanimator::reanimate(ms)); + + return fl_; + +} + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosLazyFieldLocation.h b/src/fdb5/daos/DaosLazyFieldLocation.h new file mode 100644 index 000000000..87b6bab75 --- /dev/null +++ b/src/fdb5/daos/DaosLazyFieldLocation.h @@ -0,0 +1,62 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Oct 2023 + +#pragma once + +#include "fdb5/database/FieldLocation.h" + +#include "fdb5/daos/DaosName.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// @note: used in fdb-list index visiting, in DaosIndex::entries. During +/// visitation, DaosFieldLocations are built, which normally require +/// retrieving the location information from DAOS, inflicting RPCs. +/// This DaosLazyFieldLocation, instead, remains empty and the actual +/// information is only be retrieved from DAOS when stableLocation() +/// is called. This allows the visiting mechanism to discard unmatching +/// FieldLocations before any RPC is performed for them. +class DaosLazyFieldLocation : public FieldLocation { +public: + + DaosLazyFieldLocation(const fdb5::DaosLazyFieldLocation& rhs); + DaosLazyFieldLocation(const fdb5::DaosKeyValueName& index, const std::string& key); + + eckit::DataHandle* dataHandle() const override; + + virtual std::shared_ptr make_shared() const override; + + virtual void visit(FieldLocationVisitor& visitor) const override; + + virtual std::shared_ptr stableLocation() const override; + +private: // methods + + std::unique_ptr& realise() const; + + void print(std::ostream &out) const override; + +private: // members + + fdb5::DaosKeyValueName index_; + std::string key_; + mutable std::unique_ptr fl_; + +}; + + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosName.cc b/src/fdb5/daos/DaosName.cc new file mode 100644 index 000000000..922965ed8 --- /dev/null +++ b/src/fdb5/daos/DaosName.cc @@ -0,0 +1,386 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/exception/Exceptions.h" +#include "eckit/utils/Tokenizer.h" +#include "eckit/filesystem/PathName.h" + +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosArrayHandle.h" +#include "fdb5/daos/DaosArrayPartHandle.h" +#include "fdb5/daos/DaosKeyValueHandle.h" +#include "fdb5/daos/DaosException.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosNameBase::DaosNameBase(const std::string& pool) : pool_(pool) {} + +DaosNameBase::DaosNameBase(const std::string& pool, const std::string& cont) : pool_(pool), cont_(cont) {} + +DaosNameBase::DaosNameBase(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) : pool_(pool), cont_(cont), oid_(oid) {} + +DaosNameBase::DaosNameBase(const eckit::URI& uri) { + + /// @note: uri expects a string with the following format: + /// daos:pool[/cont[/oidhi.oidlo]] + + ASSERT(uri.scheme() == "daos"); + ASSERT(uri.query() == std::string()); + ASSERT(uri.fragment() == std::string()); + + eckit::Tokenizer parse("/"); + std::vector bits; + parse(uri.name(), bits); + + ASSERT(bits.size() > 0); + ASSERT(bits.size() < 4); + + pool_ = bits[0]; + if (bits.size() > 1) cont_.emplace(bits[1]); + if (bits.size() > 2) oid_.emplace(bits[2]); + +} + +DaosNameBase::DaosNameBase(const fdb5::DaosObject& obj) : + pool_(obj.getContainer().getPool().name()), + cont_(obj.getContainer().name()), + oid_(obj.OID()) {} + +fdb5::DaosOID DaosNameBase::allocOID(const daos_otype_t& otype, const daos_oclass_id_t& oclass) const { + + ASSERT(cont_.has_value()); + ASSERT(!oid_.has_value()); + + fdb5::DaosSession s{}; + fdb5::DaosPool& p = s.getPool(pool_); + fdb5::DaosContainer& c = p.getContainer(cont_.value()); + + return fdb5::DaosOID{0, c.allocateOIDLo(), otype, oclass}; + +} + +/// @note: to be called in all DaosName* methods and DaosName* user code that intend +/// having this Name instance's OID generated (having its reserved bits populated) +/// as a post-condition. +void DaosNameBase::ensureGeneratedOID() const { + + ASSERT(cont_.has_value()); + ASSERT(oid_.has_value()); + + if (oid_.value().wasGenerated()) return; + + fdb5::DaosSession s{}; + fdb5::DaosPool& p = s.getPool(pool_); + fdb5::DaosContainer& c = p.getContainer(cont_.value()); + + oid_.value().generateReservedBits(c); + +} + +/// @todo: the otype parameter is unnecessary +std::unique_ptr DaosNameBase::buildObject(fdb5::DaosSession& s) const { + + /// @note: will throw DaosEntityNotFoundException if not exists + if (oid_->otype() == DAOS_OT_ARRAY || oid_->otype() == DAOS_OT_ARRAY_BYTE) + return std::make_unique(s, *this); + if (oid_->otype() == DAOS_OT_KV_HASHED) return std::make_unique(s, *this); + throw eckit::Exception("Provided otype not recognised."); + +} + +void DaosNameBase::create() const { + + ASSERT(cont_.has_value()); + + fdb5::DaosSession s{}; + fdb5::DaosPool& p = s.getPool(pool_); + + fdb5::DaosContainer& c = p.ensureContainer(cont_.value()); + + if (!oid_.has_value()) return; + + ensureGeneratedOID(); + + switch (oid_.value().otype()) { + case DAOS_OT_ARRAY: case DAOS_OT_ARRAY_BYTE: + c.createArray(oid_.value()); + break; + case DAOS_OT_KV_HASHED: + c.createKeyValue(oid_.value()); + break; + default: + throw eckit::Exception("Provided otype not recognised."); + } + +} + +void DaosNameBase::destroy() const { + + ASSERT(cont_.has_value()); + + fdb5::DaosSession s{}; + fdb5::DaosPool& p = s.getPool(pool_); + + /// @todo: are the semantics here OK? i.e. not throw if cont or obj not found + + if (!oid_.has_value()) { + try { + p.destroyContainer(cont_.value()); + } catch (fdb5::DaosEntityNotFoundException& e) {} + return; + } + + fdb5::DaosContainer& c = p.getContainer(cont_.value()); + + ensureGeneratedOID(); + + /// @todo: improve this code + + if (oid_->otype() == DAOS_OT_KV_HASHED) { + + try { + fdb5::DaosKeyValue kv{c, oid_.value()}; + kv.destroy(); + } catch (fdb5::DaosEntityNotFoundException& e) {} + + } else if (oid_->otype() == DAOS_OT_ARRAY || oid_->otype() == DAOS_OT_ARRAY_BYTE) { + + try { + fdb5::DaosArray a{c, oid_.value()}; + a.destroy(); + } catch (fdb5::DaosEntityNotFoundException& e) {} + + } else { + + NOTIMP; + + } + +} + +eckit::Length DaosNameBase::size() const { + + if (!oid_.has_value()) NOTIMP; + + ensureGeneratedOID(); + + fdb5::DaosSession s{}; + return eckit::Length(buildObject(s)->size()); + +} + +bool DaosNameBase::exists() const { + + fdb5::DaosSession s{}; + + try { + + fdb5::DaosPool& p = s.getPool(pool_); + + if (cont_.has_value()) + fdb5::DaosContainer& c = p.getContainer(cont_.value()); + + if (oid_.has_value()) { + ensureGeneratedOID(); + buildObject(s); + } + + } catch (const fdb5::DaosEntityNotFoundException& e) { + return false; + } + return true; + +} + +std::string DaosNameBase::asString() const { + + if (as_string_.has_value()) return as_string_.value(); + + if (oid_.has_value()) ensureGeneratedOID(); + + eckit::StringList v{pool_}; + if (cont_.has_value()) v.push_back(cont_.value()); + if (oid_.has_value()) v.push_back(oid_.value().asString()); + + std::ostringstream oss; + const char *sep = ""; + for (eckit::StringList::const_iterator j = v.begin(); j != v.end(); ++j) { + oss << sep; + oss << *j; + sep = "/"; + } + + as_string_.emplace(oss.str()); + + return as_string_.value(); + +} + +eckit::URI DaosNameBase::URI() const { + + return eckit::URI("daos", eckit::PathName(asString())); + +} + +const std::string& DaosNameBase::poolName() const { + + return pool_; + +} + +const std::string& DaosNameBase::containerName() const { + + ASSERT(cont_.has_value()); + return cont_.value(); + +} + +const fdb5::DaosOID& DaosNameBase::OID() const { + + ASSERT(oid_.has_value()); + ensureGeneratedOID(); + return oid_.value(); + +} + +bool DaosNameBase::operator<(const DaosNameBase& other) const { + return this->asString() < other.asString(); +} + +bool DaosNameBase::operator>(const DaosNameBase& other) const { + return this->asString() > other.asString(); +} + +bool DaosNameBase::operator!=(const DaosNameBase& other) const { + return this->asString() != other.asString(); +} + +bool DaosNameBase::operator==(const DaosNameBase& other) const { + return this->asString() == other.asString(); +} + +bool DaosNameBase::operator<=(const DaosNameBase& other) const { + return this->asString() <= other.asString(); +} + +bool DaosNameBase::operator>=(const DaosNameBase& other) const { + return this->asString() >= other.asString(); +} + +DaosArrayName::DaosArrayName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) : DaosNameBase(pool, cont, oid) { + + ASSERT(oid_.value().otype() == DAOS_OT_ARRAY || oid_.value().otype() == DAOS_OT_ARRAY_BYTE); + +} + +DaosArrayName::DaosArrayName(const eckit::URI& uri) : DaosNameBase(uri) { + + ASSERT(oid_.has_value()); + ASSERT(oid_.value().otype() == DAOS_OT_ARRAY || oid_.value().otype() == DAOS_OT_ARRAY_BYTE); + +} + +DaosArrayName::DaosArrayName(const fdb5::DaosArray& arr) : DaosNameBase(arr) {} + +eckit::DataHandle* DaosArrayName::dataHandle(bool overwrite) const { + + return new fdb5::DaosArrayHandle(*this); + +} + +eckit::DataHandle* DaosArrayName::dataHandle(const eckit::Offset& off, const eckit::Length& len) const { + + return new fdb5::DaosArrayPartHandle(*this, off, len); + +} + +DaosKeyValueName::DaosKeyValueName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) : DaosNameBase(pool, cont, oid) { + + ASSERT(oid_.value().otype() == DAOS_OT_KV_HASHED); + +} + +DaosKeyValueName::DaosKeyValueName(const eckit::URI& uri) : DaosNameBase(uri) { + + ASSERT(oid_.has_value()); + ASSERT(oid_.value().otype() == DAOS_OT_KV_HASHED); + +} + +DaosKeyValueName::DaosKeyValueName(const fdb5::DaosKeyValue& kv) : DaosNameBase(kv) {} + +bool DaosKeyValueName::has(const std::string& key) const { + + ASSERT(exists()); + + fdb5::DaosSession s{}; + fdb5::DaosKeyValue kv{s, *this}; + + return kv.has(key); + +} + +eckit::DataHandle* DaosKeyValueName::dataHandle(const std::string& key, bool overwrite) const { + + /// @todo: OK to serialise pointers in DaosName? + return new fdb5::DaosKeyValueHandle(*this, key); + +} + +DaosName::DaosName(const std::string& pool) : DaosNameBase(pool) {} + +DaosName::DaosName(const std::string& pool, const std::string& cont) : DaosNameBase(pool, cont) {} + +DaosName::DaosName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid) : DaosNameBase(pool, cont, oid) {} + +DaosName::DaosName(const eckit::URI& uri) : DaosNameBase(uri) {} + +DaosName::DaosName(const fdb5::DaosObject& obj) : DaosNameBase(obj) {} + +DaosArrayName DaosName::createArrayName(const daos_oclass_id_t& oclass, bool with_attr) const { + + ASSERT(hasContainerName() && !hasOID()); + create(); + if (with_attr) { + return DaosArrayName{pool_, cont_.value(), allocOID(DAOS_OT_ARRAY, oclass)}; + } else { + return DaosArrayName{pool_, cont_.value(), allocOID(DAOS_OT_ARRAY_BYTE, oclass)}; + } + +} + +DaosKeyValueName DaosName::createKeyValueName(const daos_oclass_id_t& oclass) const { + + ASSERT(hasContainerName() && !hasOID()); + create(); + return DaosKeyValueName{pool_, cont_.value(), allocOID(DAOS_OT_KV_HASHED, oclass)}; + +} + +std::vector DaosName::listOIDs() const { + + ASSERT(hasContainerName() && !hasOID()); + + fdb5::DaosSession s{}; + fdb5::DaosPool& p = s.getPool(pool_); + fdb5::DaosContainer& c = p.getContainer(cont_.value()); + + return c.listOIDs(); + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosName.h b/src/fdb5/daos/DaosName.h new file mode 100644 index 000000000..c1f1bcab8 --- /dev/null +++ b/src/fdb5/daos/DaosName.h @@ -0,0 +1,145 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Sept 2022 + +#pragma once + +#include + +#include +#include + +#include "eckit/utils/Optional.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/DataHandle.h" + +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosOID.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosSession; +class DaosPool; +class DaosContainer; + +class DaosNameBase { + +public: // methods + + /// @todo: implement DaosName::containerName to return a container DaosName, + /// and rename poolName and containerName to something else + // DaosName poolName() const; + // DaosName containerName() const; + + void ensureGeneratedOID() const; + + void create() const; + void destroy() const; + eckit::Length size() const; + bool exists() const; + // owner + // empty + + /// @todo: asString should only be used for debugging. private print? + /// @todo: asString currently ensures the OID, if present, has been generated, but + /// OID generation fails if the pool or container do not exist, and asString does + /// not succeed. A method should be implemented which supports stringifying + /// non-existing objects. + std::string asString() const; + eckit::URI URI() const; + const std::string& poolName() const; + const std::string& containerName() const; + bool hasContainerName() const { return cont_.has_value(); }; + const fdb5::DaosOID& OID() const; + bool hasOID() const { return oid_.has_value(); }; + + bool operator<(const DaosNameBase& other) const; + bool operator>(const DaosNameBase& other) const; + bool operator!=(const DaosNameBase& other) const; + bool operator==(const DaosNameBase& other) const; + bool operator<=(const DaosNameBase& other) const; + bool operator>=(const DaosNameBase& other) const; + +protected: // methods + + DaosNameBase(const std::string& pool); + DaosNameBase(const std::string& pool, const std::string& cont); + DaosNameBase(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid); + DaosNameBase(const eckit::URI&); + DaosNameBase(const fdb5::DaosObject&); + + fdb5::DaosOID allocOID(const daos_otype_t&, const daos_oclass_id_t&) const; + +private: // methods + + std::unique_ptr buildObject(fdb5::DaosSession&) const; + +protected: // members + + std::string pool_; + eckit::Optional cont_; + mutable eckit::Optional oid_; + mutable eckit::Optional as_string_; + +}; + +class DaosArrayName : public DaosNameBase { + +public: // methods + + DaosArrayName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid); + DaosArrayName(const eckit::URI&); + DaosArrayName(const fdb5::DaosArray&); + + /// @todo: think if this overwrite parameter makes sense in DAOS + eckit::DataHandle* dataHandle(bool overwrite = false) const; + eckit::DataHandle* dataHandle(const eckit::Offset&, const eckit::Length&) const; + +}; + +class DaosKeyValueName : public DaosNameBase { + +public: // methods + + DaosKeyValueName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid); + DaosKeyValueName(const eckit::URI&); + DaosKeyValueName(const fdb5::DaosKeyValue&); + + bool has(const std::string& key) const; + + /// @todo: think if this overwrite parameter makes sense in DAOS + eckit::DataHandle* dataHandle(const std::string& key, bool overwrite = false) const; + +}; + +/// @note: DaosName can represent a pool, container or object. Provides convenient functionality for containers +class DaosName : public DaosNameBase { + +public: // methods + + DaosName(const std::string& pool); + DaosName(const std::string& pool, const std::string& cont); + DaosName(const std::string& pool, const std::string& cont, const fdb5::DaosOID& oid); + DaosName(const eckit::URI&); + DaosName(const fdb5::DaosObject&); + + fdb5::DaosArrayName createArrayName(const daos_oclass_id_t& oclass = OC_S1, bool with_attr = true) const; + fdb5::DaosKeyValueName createKeyValueName(const daos_oclass_id_t& oclass = OC_S1) const; + std::vector listOIDs() const; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosOID.cc b/src/fdb5/daos/DaosOID.cc new file mode 100644 index 000000000..30eeeebf5 --- /dev/null +++ b/src/fdb5/daos/DaosOID.cc @@ -0,0 +1,149 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include +#include +#include + +#include "eckit/exception/Exceptions.h" +#include "eckit/utils/Translator.h" +#include "eckit/utils/MD5.h" + +#include "fdb5/daos/DaosOID.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosSession.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +void DaosOID::parseReservedBits() { + + otype_ = static_cast((oid_.hi & OID_FMT_TYPE_MASK) >> OID_FMT_TYPE_SHIFT); + oclass_ = OBJ_CLASS_DEF(OR_RP_1, (oid_.hi & OID_FMT_CLASS_MASK) >> OID_FMT_CLASS_SHIFT); + +} + +DaosOID::DaosOID(const uint64_t& hi, const uint64_t& lo) : oid_({lo, hi}), wasGenerated_(true) { + + parseReservedBits(); + +} + +DaosOID::DaosOID(const std::string& s) : wasGenerated_(true) { + + ASSERT(s.length() == 32); + ASSERT(std::all_of(s.begin(), s.end(), ::isxdigit)); + + oid_.hi = std::stoull(s.substr(0, 16), nullptr, 16); + oid_.lo = std::stoull(s.substr(16, 16), nullptr, 16); + + parseReservedBits(); + +} + +DaosOID::DaosOID(const uint32_t& hi, const uint64_t& lo, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass) : + otype_(otype), oid_({lo, hi}), oclass_(oclass), wasGenerated_(false) {} + +DaosOID::DaosOID(const std::string& name, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass) : + otype_(otype), oclass_(oclass), wasGenerated_(false) { + + eckit::MD5 md5(name); + /// @todo: calculate digests of 12 bytes (24 hex characters) rather than 16 bytes (32 hex characters) + /// I have tried redefining MD5_DIGEST_LENGTH but it had no effect + oid_.hi = std::stoull(md5.digest().substr(0, 8), nullptr, 16); + oid_.lo = std::stoull(md5.digest().substr(8, 16), nullptr, 16); + +} + +bool DaosOID::operator==(const DaosOID& rhs) const { + + return oid_.hi == rhs.oid_.hi && oid_.lo == rhs.oid_.lo && + oclass_ == rhs.oclass_ && wasGenerated_ == rhs.wasGenerated_; + +} + +void DaosOID::generateReservedBits(fdb5::DaosContainer& cont) { + + if (wasGenerated_) return; + + DAOS_CALL(daos_obj_generate_oid(cont.getOpenHandle(), &oid_, otype(), oclass(), 0, 0)); + + wasGenerated_ = true; + + return; + +} + +std::string DaosOID::asString() const { + + if (as_string_.has_value()) return as_string_.value(); + + ASSERT(wasGenerated_); + + std::stringstream os; + os << std::setw(16) << std::setfill('0') << std::hex << oid_.hi; + os << std::setw(16) << std::setfill('0') << std::hex << oid_.lo; + as_string_.emplace(os.str()); + return as_string_.value(); + +} + +daos_obj_id_t& DaosOID::asDaosObjIdT() { + + return oid_; + +} + +enum daos_otype_t DaosOID::otype() const { + + return otype_; + +} + +daos_oclass_id_t DaosOID::oclass() const { + + return oclass_; + +} + +DaosArrayOID::DaosArrayOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { ASSERT(otype_ == DAOS_OT_ARRAY); } + +DaosArrayOID::DaosArrayOID(const std::string& oid) : DaosOID(oid) { ASSERT(otype_ == DAOS_OT_ARRAY); } + +DaosArrayOID::DaosArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) : + DaosOID(hi, lo, DAOS_OT_ARRAY, oclass) {} + +DaosArrayOID::DaosArrayOID(const std::string& name, const daos_oclass_id_t& oclass) : + DaosOID(name, DAOS_OT_ARRAY, oclass) {} + +DaosByteArrayOID::DaosByteArrayOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { ASSERT(otype_ == DAOS_OT_ARRAY_BYTE); } + +DaosByteArrayOID::DaosByteArrayOID(const std::string& oid) : DaosOID(oid) { ASSERT(otype_ == DAOS_OT_ARRAY_BYTE); } + +DaosByteArrayOID::DaosByteArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) : + DaosOID(hi, lo, DAOS_OT_ARRAY_BYTE, oclass) {} + +DaosByteArrayOID::DaosByteArrayOID(const std::string& name, const daos_oclass_id_t& oclass) : + DaosOID(name, DAOS_OT_ARRAY_BYTE, oclass) {} + +DaosKeyValueOID::DaosKeyValueOID(const uint64_t& hi, const uint64_t& lo) : DaosOID(hi, lo) { ASSERT(otype_ == DAOS_OT_KV_HASHED); } + +DaosKeyValueOID::DaosKeyValueOID(const std::string& oid) : DaosOID(oid) { ASSERT(otype_ == DAOS_OT_KV_HASHED); } + +DaosKeyValueOID::DaosKeyValueOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass) : + DaosOID(hi, lo, DAOS_OT_KV_HASHED, oclass) {} + +DaosKeyValueOID::DaosKeyValueOID(const std::string& name, const daos_oclass_id_t& oclass) : + DaosOID(name, DAOS_OT_KV_HASHED, oclass) {} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosOID.h b/src/fdb5/daos/DaosOID.h new file mode 100644 index 000000000..e63559022 --- /dev/null +++ b/src/fdb5/daos/DaosOID.h @@ -0,0 +1,97 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Oct 2022 + +#pragma once + +#include + +#include "eckit/utils/Optional.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosContainer; + +class DaosOID { + +public: // methods + + DaosOID(const uint64_t& hi, const uint64_t& lo); + DaosOID(const std::string&); + DaosOID(const uint32_t& hi, const uint64_t& lo, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass); + DaosOID(const std::string&, const enum daos_otype_t& otype, const daos_oclass_id_t& oclass); + + bool operator==(const DaosOID&) const; + + void generateReservedBits(fdb5::DaosContainer&); + + std::string asString() const; + daos_obj_id_t& asDaosObjIdT(); + enum daos_otype_t otype() const; + daos_oclass_id_t oclass() const; + bool wasGenerated() const { return wasGenerated_; }; + +private: // methods + + void parseReservedBits(); + +protected: // members + + enum daos_otype_t otype_; + +private: // members + + daos_obj_id_t oid_; + daos_oclass_id_t oclass_; + bool wasGenerated_; + mutable eckit::Optional as_string_; + +}; + +class DaosArrayOID : public DaosOID { + +public: //methods + + DaosArrayOID(const uint64_t& hi, const uint64_t& lo); + DaosArrayOID(const std::string&); + DaosArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass); + DaosArrayOID(const std::string&, const daos_oclass_id_t& oclass); + +}; + +class DaosByteArrayOID : public DaosOID { + +public: //methods + + DaosByteArrayOID(const uint64_t& hi, const uint64_t& lo); + DaosByteArrayOID(const std::string&); + DaosByteArrayOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass); + DaosByteArrayOID(const std::string&, const daos_oclass_id_t& oclass); + +}; + +class DaosKeyValueOID : public DaosOID { + +public: //methods + + DaosKeyValueOID(const uint64_t& hi, const uint64_t& lo); + DaosKeyValueOID(const std::string&); + DaosKeyValueOID(const uint32_t& hi, const uint64_t& lo, const daos_oclass_id_t& oclass); + DaosKeyValueOID(const std::string&, const daos_oclass_id_t& oclass); + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/daos/DaosObject.cc b/src/fdb5/daos/DaosObject.cc new file mode 100644 index 000000000..2e94085f5 --- /dev/null +++ b/src/fdb5/daos/DaosObject.cc @@ -0,0 +1,484 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "eckit/exception/Exceptions.h" +#include "eckit/io/Buffer.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosException.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosObject::DaosObject(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid) : cont_(cont), oid_(oid), open_(false) { + + oid_.generateReservedBits(cont); + +} + +DaosObject::DaosObject(DaosObject&& other) noexcept : cont_(other.cont_), oid_(std::move(other.oid_)), + oh_(std::move(other.oh_)), open_(other.open_) { + + other.open_ = false; + +} + +std::string DaosObject::name() const { + + return oid_.asString(); + +} + +const fdb5::DaosOID& DaosObject::OID() const { + + return oid_; + +} + +eckit::URI DaosObject::URI() const { + + return fdb5::DaosName(*this).URI(); + +} + +fdb5::DaosContainer& DaosObject::getContainer() const { + + return cont_; + +} + +fdb5::DaosContainer& name_to_cont_ref(fdb5::DaosSession& session, const fdb5::DaosNameBase& name) { + + fdb5::UUID uuid; + + fdb5::DaosPool* pool; + if (uuid_parse(name.poolName().c_str(), uuid.internal) == 0) { + pool = &(session.getPool(uuid, name.poolName())); + } else { + pool = &(session.getPool(name.poolName())); + } + + return pool->getContainer(name.containerName()); + +} + +DaosArray::DaosArray(DaosArray&& other) noexcept : DaosObject::DaosObject(std::move(other)) {} + +DaosArray::DaosArray(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid, bool verify) : DaosObject(cont, oid) { + + ASSERT(oid_.otype() == DAOS_OT_ARRAY || oid_.otype() == DAOS_OT_ARRAY_BYTE); + + if (verify && !exists()) { + throw fdb5::DaosEntityNotFoundException( + "Array with oid " + oid.asString() + " not found", + Here()); + } + +} + +DaosArray::DaosArray(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid) : DaosArray(cont, oid, true) {} + +DaosArray::DaosArray(fdb5::DaosSession& session, const fdb5::DaosNameBase& name) : + DaosArray(name_to_cont_ref(session, name), name.OID()) {} + +DaosArray::DaosArray(fdb5::DaosSession& session, const eckit::URI& uri) : DaosArray(session, DaosName(uri)) {} + +DaosArray::~DaosArray() { + + if (open_) close(); + +} + +/// @note: non-existing arrays (DAOS_OT_ARRAY) and byte-arrays +/// (DAOS_OT_ARRAY_BYTE) are reported as existing with the current approach. +/// @todo: implement proper exist check for DAOS_OT_ARRAY_BYTE. +bool DaosArray::exists() { + + try { + + open(); + + } catch ( fdb5::DaosEntityNotFoundException& e) { + + return false; + + } + + return true; + +} + +void DaosArray::create() { + + if (open_) throw eckit::SeriousBug("Attempted create() on an open DaosArray"); + + if (oid_.otype() == DAOS_OT_ARRAY_BYTE) return open(); + + const daos_handle_t& coh = cont_.getOpenHandle(); + + DAOS_CALL( + daos_array_create( + coh, oid_.asDaosObjIdT(), DAOS_TX_NONE, + fdb5::DaosSession().objectCreateCellSize(), + fdb5::DaosSession().objectCreateChunkSize(), + &oh_, NULL + ) + ); + + open_ = true; + +} + +void DaosArray::destroy() { + + open(); + + DAOS_CALL(daos_array_destroy(oh_, DAOS_TX_NONE, NULL)); + + close(); + +} + +void DaosArray::open() { + + if (open_) return; + + const daos_handle_t& coh = cont_.getOpenHandle(); + + if (oid_.otype() == DAOS_OT_ARRAY_BYTE) { + + /// @note: when using daos_array_open_with_attr to "create" an array, + /// cell and chunk size must be specified. When opening it for + /// subsequent reads, the same values have to be provided for consistent + /// access to the written data. + /// In principle these values should be constant, so + /// inconsistencies should never happen. However if these values are + /// going to be reconfigured, they should be recorded in the database + /// on creation, and read back on database access. + DAOS_CALL( + daos_array_open_with_attr( + coh, oid_.asDaosObjIdT(), DAOS_TX_NONE, DAOS_OO_RW, + fdb5::DaosSession().objectCreateCellSize(), + fdb5::DaosSession().objectCreateChunkSize(), + &oh_, NULL + ) + ); + + } else { + + daos_size_t cell_size, csize; + + DAOS_CALL(daos_array_open(coh, oid_.asDaosObjIdT(), DAOS_TX_NONE, DAOS_OO_RW, &cell_size, &csize, &oh_, NULL)); + + } + + open_ = true; + +} + +void DaosArray::close() { + + if (!open_) { + eckit::Log::warning() << "Closing DaosArray " << name() << ", object is not open" << std::endl; + return; + } + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => daos_array_close()" << std::endl; + + int code = daos_array_close(oh_, NULL); + + if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_array_close(), file " + << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" + << code << ")" << std::endl; + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_array_close()" << std::endl; + + open_ = false; + +} + +/// @note: a buffer (buf) and its length (len) must be provided. A full write of +/// the buffer into the DAOS array will be attempted. +/// @note: daos_array_write fails if len to write is too large. +/// DaosArray::write therefore always returns a value equal to the provided +/// len if it succeeds (i.e. if no exception is thrown). +uint64_t DaosArray::write(const void* buf, const uint64_t& len, const eckit::Offset& off) { + + open(); + + daos_array_iod_t iod; + daos_range_t rg; + + d_sg_list_t sgl; + d_iov_t iov; + + iod.arr_nr = 1; + rg.rg_len = (daos_size_t) len; + rg.rg_idx = (daos_off_t) off; + iod.arr_rgs = &rg; + + sgl.sg_nr = 1; + d_iov_set(&iov, (void*) buf, (size_t) len); + sgl.sg_iovs = &iov; + + DAOS_CALL(daos_array_write(oh_, DAOS_TX_NONE, &iod, &sgl, NULL)); + + return len; + +} + +/// @note: a buffer (buf) and its length (len) must be provided. A read from the +/// DAOS array of the full buffer length will be attempted. +/// @note: daos_array_read does not fail if requested len is larger than object, +/// and does not report back the actual amount of bytes read. The user must +/// call DaosArray::size() (i.e. daos_array_get_size) first and request +/// read of a correspondingly sized buffer. +/// @todo: implement the note above in dummy DAOS. +/// @note: therefore, this method won't be compatible with the DataHandle::read +/// interface (which is expected to return actual read bytes) unless the +/// implementation of such method checks the actual size and returns it +/// if smaller than the provided buffer - which would entail an additional +/// RPC and reduce performance. If the read interface is not properly +/// implemented, at least DataHandle::copyTo breaks. +/// @todo: fix this issue by using the "short_read" feature in daos_array_read +/// and returning actual read size, rather han having DataHandle::read +/// check the size. +/// @note: see DaosArrayPartHandle for cases where the object size is known. +uint64_t DaosArray::read(void* buf, uint64_t len, const eckit::Offset& off) { + + open(); + + daos_array_iod_t iod; + daos_range_t rg; + + d_sg_list_t sgl; + d_iov_t iov; + + iod.arr_nr = 1; + rg.rg_len = len; + rg.rg_idx = (daos_off_t) off; + iod.arr_rgs = &rg; + + sgl.sg_nr = 1; + d_iov_set(&iov, buf, (size_t) len); + sgl.sg_iovs = &iov; + + iod.arr_nr_short_read = 1; + + DAOS_CALL(daos_array_read(oh_, DAOS_TX_NONE, &iod, &sgl, NULL)); + + return len; + +} + +uint64_t DaosArray::size() { + + open(); + + uint64_t array_size; + + DAOS_CALL(daos_array_get_size(oh_, DAOS_TX_NONE, &array_size, NULL)); + + return array_size; + +} + +DaosKeyValue::DaosKeyValue(DaosKeyValue&& other) noexcept : DaosObject::DaosObject(std::move(other)) {} + +DaosKeyValue::DaosKeyValue(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid, bool verify) : DaosObject(cont, oid) { + + ASSERT(oid_.otype() == type()); + + if (verify && !exists()) { + throw fdb5::DaosEntityNotFoundException( + "KeyValue with oid " + oid.asString() + " not found", + Here()); + } + +} + +DaosKeyValue::DaosKeyValue(fdb5::DaosContainer& cont, const fdb5::DaosOID& oid) : DaosKeyValue(cont, oid, true) {} + +DaosKeyValue::DaosKeyValue(fdb5::DaosSession& session, const fdb5::DaosNameBase& name) : + DaosKeyValue(name_to_cont_ref(session, name), name.OID()) {} + +DaosKeyValue::DaosKeyValue(fdb5::DaosSession& session, const eckit::URI& uri) : DaosKeyValue(session, DaosName(uri)) {} + +DaosKeyValue::~DaosKeyValue() { + + if (open_) close(); + +} + +bool DaosKeyValue::exists() { + + open(); /// @note: creates it if not exists + + return true; + +} + +void DaosKeyValue::create() { + + if (open_) throw eckit::SeriousBug("Attempted create() on an open DaosKeyValue"); + + const daos_handle_t& coh = cont_.getOpenHandle(); + + DAOS_CALL(daos_kv_open(coh, oid_.asDaosObjIdT(), DAOS_OO_RW, &oh_, NULL)); + + open_ = true; + +} + +void DaosKeyValue::destroy() { + + open(); + + DAOS_CALL(daos_kv_destroy(oh_, DAOS_TX_NONE, NULL)); + + close(); + +} + +void DaosKeyValue::open() { + + if (open_) return; + + create(); + +} + +void DaosKeyValue::close() { + + if (!open_) { + eckit::Log::warning() << "Closing DaosKeyValue " << name() << ", object is not open" << std::endl; + return; + } + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => daos_obj_close()" << std::endl; + + int code = daos_obj_close(oh_, NULL); + + if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_obj_close(), file " + << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" + << code << ")" << std::endl; + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_obj_close()" << std::endl; + + open_ = false; + +} + +uint64_t DaosKeyValue::size(const std::string& key) { + + open(); + + uint64_t res{0}; + + DAOS_CALL(daos_kv_get(oh_, DAOS_TX_NONE, 0, key.c_str(), &res, nullptr, NULL)); + + return res; + +} + +bool DaosKeyValue::has(const std::string& key) { + + return size(key) != 0; + +} + +/// @note: daos_kv_put fails if written len is smaller than requested len +uint64_t DaosKeyValue::put(const std::string& key, const void* buf, const uint64_t& len) { + + open(); + + DAOS_CALL(daos_kv_put(oh_, DAOS_TX_NONE, 0, key.c_str(), len, buf, NULL)); + + return len; + +} + +/// @note: daos_kv_get fails if requested value does not fit in buffer +uint64_t DaosKeyValue::get(const std::string& key, void* buf, const uint64_t& len) { + + open(); + + uint64_t res{len}; + + DAOS_CALL(daos_kv_get(oh_, DAOS_TX_NONE, 0, key.c_str(), &res, buf, NULL)); + + if (res == 0) throw fdb5::DaosEntityNotFoundException("Key '" + key + "' not found in KeyValue with OID " + oid_.asString()); + + return res; + +} + +void DaosKeyValue::remove(const std::string& key) { + + open(); + + DAOS_CALL(daos_kv_remove(oh_, DAOS_TX_NONE, 0, key.c_str(), NULL)); + +} + +std::vector DaosKeyValue::keys() { + + /// @todo: proper memory management + int max_keys_per_rpc = 1024; /// @todo: take from config + daos_key_desc_t key_sizes[max_keys_per_rpc]; + d_sg_list_t sgl; + d_iov_t sg_iov; + size_t bufsize = 1024; + eckit::Buffer list_buf{bufsize}; + d_iov_set(&sg_iov, (char*) list_buf, bufsize); + sgl.sg_nr = 1; + sgl.sg_nr_out = 0; + sgl.sg_iovs = &sg_iov; + daos_anchor_t listing_status = DAOS_ANCHOR_INIT; + std::vector listed_keys; + + while (!daos_anchor_is_eof(&listing_status)) { + uint32_t nkeys_found = max_keys_per_rpc; + int rc; + list_buf.zero(); + + DAOS_CALL(daos_kv_list(oh_, DAOS_TX_NONE, &nkeys_found, key_sizes, &sgl, &listing_status, NULL)); + + size_t key_start = 0; + for (int i = 0; i < nkeys_found; i++) { + listed_keys.push_back(std::string((char*) list_buf + key_start, key_sizes[i].kd_key_len)); + key_start += key_sizes[i].kd_key_len; + } + } + return listed_keys; + +} + +eckit::MemoryStream DaosKeyValue::getMemoryStream(std::vector& v, const std::string& key, const std::string& kvTitle) { + + uint64_t sz = size(key); + if (sz == 0) throw fdb5::DaosEntityNotFoundException(std::string("Key '") + key + "' not found in " + kvTitle); + v.resize(sz); + get(key, &v[0], sz); + + return eckit::MemoryStream(&v[0], sz); + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosObject.h b/src/fdb5/daos/DaosObject.h new file mode 100644 index 000000000..b623788a9 --- /dev/null +++ b/src/fdb5/daos/DaosObject.h @@ -0,0 +1,151 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jul 2022 + +#pragma once + +#include + +#include + +#include "eckit/utils/Optional.h" +#include "eckit/filesystem/URI.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/serialisation/MemoryStream.h" + +#include "fdb5/daos/DaosOID.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosNameBase; + +class DaosContainer; + +class DaosSession; + +class DaosObject { + + friend DaosContainer; + +public: // methods + + DaosObject(DaosObject&&) noexcept; + DaosObject(const DaosObject&) = default; + + virtual ~DaosObject() = default; + + virtual daos_otype_t type() const = 0; + + virtual bool exists() = 0; + virtual void destroy() = 0; + virtual void open() = 0; + virtual void close() = 0; + virtual uint64_t size() = 0; + + std::string name() const; + const fdb5::DaosOID& OID() const; + eckit::URI URI() const; + fdb5::DaosContainer& getContainer() const; + +protected: // methods + + DaosObject(fdb5::DaosContainer&, const fdb5::DaosOID&); + +private: // methods + + virtual void create() = 0; + +protected: // members + + fdb5::DaosContainer& cont_; + fdb5::DaosOID oid_; + daos_handle_t oh_; + bool open_; + +}; + +class DaosArray : public DaosObject { + + friend DaosContainer; + +public: // methods + + DaosArray(DaosArray&&) noexcept; + + DaosArray(fdb5::DaosContainer&, const fdb5::DaosOID&); + DaosArray(fdb5::DaosSession&, const fdb5::DaosNameBase&); + DaosArray(fdb5::DaosSession&, const eckit::URI&); + + ~DaosArray(); + + daos_otype_t type() const override { return OID().otype(); } + bool exists() override; + void destroy() override; + void open() override; + void close() override; + uint64_t size() override; + + uint64_t write(const void*, const uint64_t&, const eckit::Offset&); + uint64_t read(void*, uint64_t, const eckit::Offset&); + +private: // methods + + DaosArray(fdb5::DaosContainer&, const fdb5::DaosOID&, bool verify); + + void create() override; + +}; + +class DaosKeyValue : public DaosObject { + + friend DaosContainer; + +public: // methods + + DaosKeyValue(DaosKeyValue&&) noexcept; + + DaosKeyValue(fdb5::DaosContainer&, const fdb5::DaosOID&); + DaosKeyValue(fdb5::DaosSession&, const fdb5::DaosNameBase&); + DaosKeyValue(fdb5::DaosSession&, const eckit::URI&); + + ~DaosKeyValue(); + + daos_otype_t type() const override { return DAOS_OT_KV_HASHED; } + bool exists() override; + void destroy() override; + void open() override; + void close() override; + uint64_t size() override { NOTIMP; }; + + uint64_t size(const std::string& key); + bool has(const std::string& key); + uint64_t put(const std::string& key, const void*, const uint64_t&); + uint64_t get(const std::string& key, void*, const uint64_t&); + void remove(const std::string& key); + std::vector keys(); + + /// @note: expects empty vector + eckit::MemoryStream getMemoryStream(std::vector& v, const std::string& key, const std::string& kvTitle = "kv"); + +private: // methods + + DaosKeyValue(fdb5::DaosContainer&, const fdb5::DaosOID&, bool verify); + + void create() override; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosPool.cc b/src/fdb5/daos/DaosPool.cc new file mode 100644 index 000000000..05d5d0f5a --- /dev/null +++ b/src/fdb5/daos/DaosPool.cc @@ -0,0 +1,342 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/exception/Exceptions.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosException.h" + +#ifdef fdb5_HAVE_DAOS_ADMIN +extern "C" { +#include +} +#endif + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosPool::DaosPool(DaosPool&& other) noexcept : uuid_(std::move(other.uuid_)), + known_uuid_(other.known_uuid_), label_(std::move(other.label_)), + poh_(std::move(other.poh_)), open_(other.open_), + cont_cache_(std::move(other.cont_cache_)) { + + other.open_ = false; + +} + +DaosPool::DaosPool() : known_uuid_(false), open_(false) {} + +DaosPool::DaosPool(const fdb5::UUID& uuid) : uuid_(uuid), known_uuid_(true), open_(false) {} + +DaosPool::DaosPool(const std::string& label) : known_uuid_(false), label_(label), open_(false) {} + + +DaosPool::DaosPool(const fdb5::UUID& uuid, const std::string& label) : + uuid_(uuid), known_uuid_(true), label_(label), open_(false) {} + +DaosPool::~DaosPool() { + + cont_cache_.clear(); + + if (open_) close(); + +} + +DaosPool& DaosPool::operator=(DaosPool&& other) noexcept { + + uuid_ = std::move(other.uuid_); + known_uuid_ = other.known_uuid_; + label_ = std::move(other.label_); + poh_ = std::move(other.poh_); + open_ = other.open_; + cont_cache_ = std::move(other.cont_cache_); + + other.open_ = false; + + return *this; + +} + +#ifdef fdb5_HAVE_DAOS_ADMIN +void DaosPool::create(const uint64_t& scmSize, const uint64_t& nvmeSize) { + + /// @note: cannot create a pool with a user-specified UUID + ASSERT(!known_uuid_); + + /// @todo: ensure deallocation. Either try catch or make a wrapper. + /// @todo: rename where possible to make less obscure + d_rank_list_t svcl; + svcl.rl_nr = 1; + D_ALLOC_ARRAY(svcl.rl_ranks, svcl.rl_nr); + ASSERT(svcl.rl_ranks); + + daos_prop_t *prop = NULL; + + if (label_.size() > 0) { + + /// @todo: Ensure freeing. default_delete. + // Throw exception if that fails. + prop = daos_prop_alloc(1); + prop->dpp_entries[0].dpe_type = DAOS_PROP_PO_LABEL; + D_STRNDUP(prop->dpp_entries[0].dpe_str, label_.c_str(), DAOS_PROP_LABEL_MAX_LEN); + + } + + DAOS_CALL( + dmg_pool_create( + fdb5::DaosSession().dmgConfigFile().c_str(), geteuid(), getegid(), NULL, NULL, + scmSize, nvmeSize, + prop, &svcl, uuid_.internal + ) + ); + + /// @todo: query the pool to ensure it's ready + + if (prop != NULL) daos_prop_free(prop); + + D_FREE(svcl.rl_ranks); + + known_uuid_ = true; + +} +#endif + +void DaosPool::open() { + + if (open_) return; + + ASSERT(known_uuid_ || label_.size() > 0); + + if (label_.size() > 0) { + DAOS_CALL(daos_pool_connect(label_.c_str(), NULL, DAOS_PC_RW, &poh_, NULL, NULL)); + } else { + char uuid_cstr[37] = ""; + uuid_unparse(uuid_.internal, uuid_cstr); + DAOS_CALL(daos_pool_connect(uuid_cstr, NULL, DAOS_PC_RW, &poh_, NULL, NULL)); + } + + open_ = true; + +} + +void DaosPool::close() { + + if (!open_) { + eckit::Log::warning() << "Disconnecting DaosPool " << name() << ", pool is not open" << std::endl; + return; + } + + closeContainers(); + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => daos_pool_disconnect()" << std::endl; + + int code = daos_pool_disconnect(poh_, NULL); + + if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_pool_disconnect(), file " + << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" + << code << ")" << std::endl; + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_pool_disconnect()" << std::endl; + + open_ = false; + +} + +std::map::iterator DaosPool::getCachedContainer(const std::string& label) { + + return cont_cache_.find(label); + +} + +fdb5::DaosContainer& DaosPool::getContainer(const std::string& label, bool verify) { + + std::map::iterator it = getCachedContainer(label); + + if (it != cont_cache_.end()) return it->second; + + fdb5::DaosContainer c{*this, label}; + + if (verify && !c.exists()) { + throw fdb5::DaosEntityNotFoundException( + "Container with label " + label + " not found", + Here()); + } + + return cont_cache_.emplace(label, std::move(c)).first->second; + +} + +fdb5::DaosContainer& DaosPool::getContainer(const std::string& label) { + + return getContainer(label, true); + +} + +fdb5::DaosContainer& DaosPool::createContainer(const std::string& label) { + + fdb5::DaosContainer& c = getContainer(label, false); + + /// @todo: if the container is found in the cache, it can be assumed + /// it exists and this creation could be skipped + c.create(); + + return c; + +} + +fdb5::DaosContainer& DaosPool::ensureContainer(const std::string& label) { + + fdb5::DaosContainer& c = getContainer(label, false); + + /// @todo: if the container is found in the cache, it can be assumed + /// it exists and this check could be skipped + if (c.exists()) return c; + + c.create(); + + return c; + +} + +void DaosPool::destroyContainer(const std::string& label) { + + ASSERT(label.size() > 0); + + /// @todo: consider cases where DaosContainer instances may be cached + /// which only have a uuid and no label, but correspond to the same + /// container being destroyed here + + bool found = false; + + std::map::iterator it = getCachedContainer(label); + + if (it != cont_cache_.end()) { + + cont_cache_.erase(it); + + } else { + + throw fdb5::DaosEntityNotFoundException( + "Container with label " + label + " not found", + Here()); + + } + + DAOS_CALL(daos_cont_destroy(poh_, label.c_str(), 1, NULL)); + + /// @todo: whenever a container is destroyed and the user owns open objects, their handles may not + /// be possible to close anymore, and any actions on such objects will fail + +} + +/// @note: indended for DaosPool::close() +void DaosPool::closeContainers() { + + std::map::iterator it; + for (it = cont_cache_.begin(); it != cont_cache_.end(); ++it) it->second.close(); + +} + +std::vector DaosPool::listContainers() { + + std::vector res; + + daos_size_t ncont{0}; + DAOS_CALL(daos_pool_list_cont(getOpenHandle(), &ncont, NULL, NULL)); + + if (ncont == 0) return res; + + std::vector info(ncont); + DAOS_CALL(daos_pool_list_cont(getOpenHandle(), &ncont, info.data(), NULL)); + + for (const auto& c : info) res.push_back(c.pci_label); + + return res; + +} + +const daos_handle_t& DaosPool::getOpenHandle() { + + open(); + return poh_; + +} + +bool DaosPool::exists() { + + /// @todo: implement this with more appropriate DAOS API functions + try { + + open(); + + } catch (eckit::SeriousBug& e) { + + return false; + + } + + return true; + +} + +std::string DaosPool::name() const { + + /// @note: cannot generate a name for an unidentified pool. + /// Either create it or provide a label upon construction. + ASSERT(label_.size() > 0 || known_uuid_); + + if (label_.size() > 0) return label_; + + char name_cstr[37]; + uuid_unparse(uuid_.internal, name_cstr); + return std::string(name_cstr); + +} + +const fdb5::UUID& DaosPool::uuid() const { + + return uuid_; + +} + +std::string DaosPool::label() const { + + return label_; + +} + +//---------------------------------------------------------------------------------------------------------------------- + +#ifdef fdb5_HAVE_DAOS_ADMIN +AutoPoolDestroy::~AutoPoolDestroy() noexcept(false) { + + bool fail = !eckit::Exception::throwing(); + + try { + fdb5::DaosSession().destroyPool(pool_.uuid()); + } catch (std::exception& e) { + eckit::Log::error() << "** " << e.what() << " Caught in " << Here() << std::endl; + if (fail) { + eckit::Log::error() << "** Exception is re-thrown" << std::endl; + throw; + } + eckit::Log::error() << "** An exception is already in progress" << std::endl; + eckit::Log::error() << "** Exception is ignored" << std::endl; + } + +} +#endif + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosPool.h b/src/fdb5/daos/DaosPool.h new file mode 100644 index 000000000..f9fb24008 --- /dev/null +++ b/src/fdb5/daos/DaosPool.h @@ -0,0 +1,113 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jul 2022 + +#pragma once + +#include + +#include +#include + +#include "fdb5/daos/UUID.h" +#include "fdb5/daos/DaosContainer.h" + +#include "fdb5/fdb5_config.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +using ContainerCache = std::map; + +class DaosSession; + +class DaosPool { + +public: // methods + + DaosPool(DaosPool&&) noexcept; + ~DaosPool(); + + DaosPool& operator=(DaosPool&&) noexcept; + + void open(); + void close(); + + fdb5::DaosContainer& getContainer(const std::string&); + + fdb5::DaosContainer& createContainer(const std::string&); + + fdb5::DaosContainer& ensureContainer(const std::string&); + + void destroyContainer(const std::string&); + + std::vector listContainers(); + + const daos_handle_t& getOpenHandle(); + + std::string name() const; + const fdb5::UUID& uuid() const; + std::string label() const; + +private: // methods + + friend class DaosSession; + + DaosPool(); + DaosPool(const fdb5::UUID&); + DaosPool(const std::string&); + DaosPool(const fdb5::UUID&, const std::string&); + +#ifdef fdb5_HAVE_DAOS_ADMIN + void create(const uint64_t& scmSize, const uint64_t& nvmeSize); +#endif + + fdb5::DaosContainer& getContainer(const std::string&, bool); + + void closeContainers(); + + bool exists(); + + ContainerCache::iterator getCachedContainer(const std::string&); + +private: // members + + fdb5::UUID uuid_; + bool known_uuid_; + std::string label_ = std::string(); + daos_handle_t poh_; + bool open_; + + ContainerCache cont_cache_; + +}; + +#ifdef fdb5_HAVE_DAOS_ADMIN +class AutoPoolDestroy { + +public: // methods + + AutoPoolDestroy(fdb5::DaosPool& pool) : pool_(pool) {} + + ~AutoPoolDestroy() noexcept(false); + +private: // members + + fdb5::DaosPool& pool_; + +}; +#endif + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosSession.cc b/src/fdb5/daos/DaosSession.cc new file mode 100644 index 000000000..98221bcc9 --- /dev/null +++ b/src/fdb5/daos/DaosSession.cc @@ -0,0 +1,272 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include + +#include "eckit/runtime/Main.h" +#include "eckit/utils/Translator.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosException.h" + +#ifdef fdb5_HAVE_DAOS_ADMIN +extern "C" { +#include +} +#endif + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosManager::DaosManager() : + containerOidsPerAlloc_(100), + objectCreateCellSize_(1), + objectCreateChunkSize_(1048576) { + +#ifdef fdb5_HAVE_DAOS_ADMIN + dmgConfigFile_ = eckit::Resource( + "fdbDaosDmgConfigFile;$FDB_DAOS_DMG_CONFIG_FILE", dmgConfigFile_ + ); +#endif + + /// @note: daos_init can be called multiple times. An internal reference + /// count is maintained by the library + DAOS_CALL(daos_init()); + +} + +DaosManager::~DaosManager() { + + pool_cache_.clear(); + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => daos_fini()" << std::endl; + + int code = daos_fini(); + + if (code < 0) eckit::Log::warning() << "DAOS error in call to daos_fini(), file " + << __FILE__ << ", line " << __LINE__ << ", function " << __func__ << " [" << code << "] (" + << code << ")" << std::endl; + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= daos_fini()" << std::endl; + +} + +void DaosManager::error(int code, const char* msg, const char* file, int line, const char* func) { + + std::ostringstream oss; + oss << "DAOS error " << msg << ", file " << file << ", line " << line << ", function " << func << " [" << code + << "] (" << code << ")"; + throw eckit::SeriousBug(oss.str()); + +} + +void DaosManager::configure(const eckit::LocalConfiguration& config) { + + std::lock_guard lock(mutex_); + containerOidsPerAlloc_ = config.getInt("container_oids_per_alloc", containerOidsPerAlloc_); + objectCreateCellSize_ = config.getInt64("object_create_cell_size", objectCreateCellSize_); + objectCreateChunkSize_ = config.getInt64("object_create_chunk_size", objectCreateChunkSize_); + +#ifdef fdb5_HAVE_DAOS_ADMIN + dmgConfigFile_ = config.getString("dmg_config_file", dmgConfigFile_); +#endif + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +DaosSession::DaosSession() : + // mutex_(DaosManager::instance().mutex_), + pool_cache_(DaosManager::instance().pool_cache_) {} + +/// @todo: make getCachedPool return a *DaosPool rather than an iterator? +/// and check it == nullptr rather than it == pool_cache_.end(). +std::deque::iterator DaosSession::getCachedPool(const fdb5::UUID& uuid) { + + std::deque::iterator it; + for (it = pool_cache_.begin(); it != pool_cache_.end(); ++it) { + + if (uuid_compare(uuid.internal, it->uuid().internal) == 0) break; + + } + + return it; + +} + +std::deque::iterator DaosSession::getCachedPool(const std::string& label) { + + std::deque::iterator it; + for (it = pool_cache_.begin(); it != pool_cache_.end(); ++it) { + + if (it->label() == label) break; + + } + + return it; + +} + +#ifdef fdb5_HAVE_DAOS_ADMIN +fdb5::DaosPool& DaosSession::createPool(const uint64_t& scmSize, const uint64_t& nvmeSize) { + + pool_cache_.push_front(fdb5::DaosPool()); + + fdb5::DaosPool& p = pool_cache_.at(0); + + p.create(scmSize, nvmeSize); + + return p; + +} + +fdb5::DaosPool& DaosSession::createPool(const std::string& label, const uint64_t& scmSize, const uint64_t& nvmeSize) { + + pool_cache_.push_front(fdb5::DaosPool(label)); + + fdb5::DaosPool& p = pool_cache_.at(0); + + p.create(scmSize, nvmeSize); + + return p; + +} +#endif + +fdb5::DaosPool& DaosSession::getPool(const fdb5::UUID& uuid) { + + std::deque::iterator it = getCachedPool(uuid); + + if (it != pool_cache_.end()) return *it; + + fdb5::DaosPool p(uuid); + + if (!p.exists()) { + char uuid_cstr[37]; + uuid_unparse(uuid.internal, uuid_cstr); + std::string uuid_str(uuid_cstr); + throw fdb5::DaosEntityNotFoundException( + "Pool with uuid " + uuid_str + " not found", + Here()); + } + + pool_cache_.push_front(std::move(p)); + + return pool_cache_.at(0); + +} + +fdb5::DaosPool& DaosSession::getPool(const std::string& label) { + + std::deque::iterator it = getCachedPool(label); + + if (it != pool_cache_.end()) return *it; + + fdb5::DaosPool p(label); + + if (!p.exists()) { + throw fdb5::DaosEntityNotFoundException( + "Pool with label " + label + " not found", + Here()); + } + + pool_cache_.push_front(std::move(p)); + + return pool_cache_.at(0); + +} + +DaosPool& DaosSession::getPool(const fdb5::UUID& uuid, const std::string& label) { + + /// @note: When both pool uuid and label are known, using this method + /// to declare a pool is preferred to avoid the following + /// inconsistencies and/or inefficiencies: + /// - when a user declares a pool by label in a process where that pool + /// has not been created, that pool will live in the cache with only a + /// label and no uuid. If the user later declares the same pool from its + /// uuid, two DaosPool instances will exist in the cache for the same + /// DAOS pool, each with their connection handle. + /// - these two instances will be incomplete and the user may not be able + /// to retrieve the label/uuid information. + + std::deque::iterator it = getCachedPool(uuid); + if (it != pool_cache_.end()) { + + if (it->label() == label) return *it; + + pool_cache_.push_front(fdb5::DaosPool(uuid, label)); + return pool_cache_.at(0); + + } + + it = getCachedPool(label); + if (it != pool_cache_.end()) return *it; + + fdb5::DaosPool p(uuid, label); + + if (!p.exists()) { + char uuid_cstr[37]; + uuid_unparse(uuid.internal, uuid_cstr); + std::string uuid_str(uuid_cstr); + throw fdb5::DaosEntityNotFoundException( + "Pool with uuid " + uuid_str + " or label " + label + " not found", + Here()); + } + + pool_cache_.push_front(std::move(p)); + + return pool_cache_.at(0); + +} + +#ifdef fdb5_HAVE_DAOS_ADMIN + +void DaosSession::destroyPool(const fdb5::UUID& uuid, const int& force) { + + bool found = false; + std::string label = std::string(); + + std::deque::iterator it = pool_cache_.begin(); + while (it != pool_cache_.end()) { + + if (uuid_compare(uuid.internal, it->uuid().internal) == 0) { + found = true; + label = it->label(); + /// @todo: should destroy pool containers here? + it = pool_cache_.erase(it); + } else { + ++it; + } + + } + + if (!found) { + + char uuid_cstr[37]; + uuid_unparse(uuid.internal, uuid_cstr); + std::string uuid_str(uuid_cstr); + throw fdb5::DaosEntityNotFoundException( + "Pool with uuid " + uuid_str + " not found", + Here()); + + } + + DAOS_CALL(dmg_pool_destroy(dmgConfigFile().c_str(), uuid.internal, NULL, force)); + + /// @todo: cached DaosPools declared with a label only, pointing to the pool + // being destroyed may still exist and should be closed and removed + +} +#endif + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosSession.h b/src/fdb5/daos/DaosSession.h new file mode 100644 index 000000000..f8f5ae1f1 --- /dev/null +++ b/src/fdb5/daos/DaosSession.h @@ -0,0 +1,169 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jul 2022 + +#pragma once + +#include + +#include +#include +#include + +#include "eckit/exception/Exceptions.h" +#include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/log/Timer.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosException.h" + +#include "fdb5/fdb5_config.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +/// @todo: Use std::map +/// @todo: Offload caching to manager? +using PoolCache = std::deque; + +/// @todo: move to a separate file +class DaosManager : private eckit::NonCopyable { + +public: // methods + + static DaosManager& instance() { + static DaosManager instance; + return instance; + }; + + static void error(int code, const char* msg, const char* file, int line, const char* func); + + void configure(const eckit::LocalConfiguration&); + +private: // methods + + DaosManager(); + + ~DaosManager(); + +private: // members + + friend class DaosSession; + + std::mutex mutex_; + PoolCache pool_cache_; + + /// @note: sets number of OIDs allocated in a single daos_cont_alloc_oids call + int containerOidsPerAlloc_; + /// @note: number of bytes per cell in a DAOS object + /// @note: cell size is the unit of atomicity / update. If the object + /// will always be updated or read in elements of e.g. 64k, then 64k + /// can be used as the cell size. Then, that 64k element cannot be + /// partially updated anymore. + uint64_t objectCreateCellSize_; + /// @note: number of cells of per dkey in a DAOS object + /// @note: the chunk size maps to how many cells to put under 1 dkey. + /// So it also controls the RPC size. It should not really be something + /// very small otherwise it might create a lot of RPCs. If not + /// using redundancy (SX), setting it to something as equal or a multiple + /// of the most common transfer size is OK. the default is 1 MiB which is + /// usually OK. If using EC, it gets more tricky as the EC cell size and + /// transfer size come into play and break that even more and can cause + /// overhead on the client side. Ideally, set both to the same as the IO + /// size, but that is not always possible because the EC cell size is + /// only changed per container, vs the chunk and transfer size that can + /// vary per object / file. That may be changed in DAOS to allow more + /// flexibility + uint64_t objectCreateChunkSize_; + +#ifdef fdb5_HAVE_DAOS_ADMIN + std::string dmgConfigFile_; +#endif + +}; + +#define DAOS_CALL(a) fdb5::daos_call(a, #a, __FILE__, __LINE__, __func__) + +static inline int daos_call(int code, const char* msg, const char* file, int line, const char* func) { + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL => " << msg << std::endl; + + if (code < 0) { + eckit::Log::error() << "DAOS_FAIL !! " << msg << std::endl; + if (code == -DER_NONEXIST) throw fdb5::DaosEntityNotFoundException(msg); + if (code == -DER_EXIST) throw fdb5::DaosEntityAlreadyExistsException(msg); + DaosManager::error(code, msg, file, line, func); + } + + LOG_DEBUG_LIB(LibFdb5) << "DAOS_CALL <= " << msg << std::endl; + + return code; +} + +//---------------------------------------------------------------------------------------------------------------------- + +/// @note: DaosSession acts as a mere wrapper for DaosManager such that DaosManager::instance does not need +/// to be called in so many places +/// @note: DaosSession no longer performs daos_init on creation and daos_fini on destroy. This is because +/// any pool handles obtained within a session are cached in DaosManager beyond DaosSession lifetime, +/// and the pool handles may become invalid if daos_fini is called for all sessions + +class DaosSession : eckit::NonCopyable { + +public: // methods + + DaosSession(); + ~DaosSession() {}; + +#ifdef fdb5_HAVE_DAOS_ADMIN + // administrative + fdb5::DaosPool& createPool( + const uint64_t& scmSize = 10ULL << 30, + const uint64_t& nvmeSize = 40ULL << 30); + fdb5::DaosPool& createPool( + const std::string& label, + const uint64_t& scmSize = 10ULL << 30, + const uint64_t& nvmeSize = 40ULL << 30); + void destroyPool(const fdb5::UUID&, const int& force = 1); +#endif + + fdb5::DaosPool& getPool(const fdb5::UUID&); + fdb5::DaosPool& getPool(const std::string&); + fdb5::DaosPool& getPool(const fdb5::UUID&, const std::string&); + + int containerOidsPerAlloc() const { return DaosManager::instance().containerOidsPerAlloc_; }; + uint64_t objectCreateCellSize() const { return DaosManager::instance().objectCreateCellSize_; }; + uint64_t objectCreateChunkSize() const { return DaosManager::instance().objectCreateChunkSize_; }; + +#ifdef fdb5_HAVE_DAOS_ADMIN + std::string dmgConfigFile() const { return DaosManager::instance().dmgConfigFile_; }; +#endif + +private: // methods + + PoolCache::iterator getCachedPool(const fdb5::UUID&); + PoolCache::iterator getCachedPool(const std::string&); + +private: // members + + /// @todo: add lock_guards in all DaosSession methods using pool_cache_. Same for cont_cache_ in DaosPool. + // std::mutex& mutex_; + PoolCache& pool_cache_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosStore.cc b/src/fdb5/daos/DaosStore.cc new file mode 100644 index 000000000..697d91571 --- /dev/null +++ b/src/fdb5/daos/DaosStore.cc @@ -0,0 +1,174 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/config/Resource.h" + +#include "fdb5/daos/DaosFieldLocation.h" +#include "fdb5/daos/DaosStore.h" +#include "fdb5/daos/DaosArrayHandle.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosException.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosStore::DaosStore(const Schema& schema, const Key& key, const Config& config) : + Store(schema), DaosCommon(config, "store", key), db_str_(db_cont_) {} + +eckit::URI DaosStore::uri() const { + + return fdb5::DaosName(pool_, db_str_).URI(); + +} + +bool DaosStore::uriBelongs(const eckit::URI& uri) const { + + /// @todo: avoid building a DaosName as it makes uriBelongs expensive + /// @todo: assert uri points to a (not necessarily existing) array object + return ( + (uri.scheme() == type()) && + (fdb5::DaosName(uri).containerName().rfind(db_str_, 0) == 0)); + +} + +bool DaosStore::uriExists(const eckit::URI& uri) const { + + /// @todo: revisit the name of this method + + ASSERT(uri.scheme() == type()); + fdb5::DaosName n(uri); + ASSERT(n.hasContainerName()); + ASSERT(n.poolName() == pool_); + ASSERT(n.containerName() == db_str_); + ASSERT(n.hasOID()); + + return n.exists(); + +} + +std::vector DaosStore::collocatedDataURIs() const { + + std::vector collocated_data_uris; + + fdb5::DaosName db_cont{pool_, db_str_}; + + if (!db_cont.exists()) return collocated_data_uris; + + for (const auto& oid : db_cont.listOIDs()) { + + if (oid.otype() != DAOS_OT_KV_HASHED && oid.otype() != DAOS_OT_ARRAY_BYTE) + throw eckit::SeriousBug("Found non-KV non-ByteArray objects in DB container " + db_cont.URI().asString()); + + if (oid.otype() == DAOS_OT_KV_HASHED) continue; + + collocated_data_uris.push_back(fdb5::DaosArrayName(pool_, db_str_, oid).URI()); + + } + + return collocated_data_uris; + +} + +std::set DaosStore::asCollocatedDataURIs(const std::vector& uris) const { + + std::set res; + + for (auto& uri : uris) + /// @note: seems redundant, but intends to check validity of input URIs + res.insert(fdb5::DaosName(uri).URI()); + + return res; + +} + +bool DaosStore::exists() const { + + return fdb5::DaosName(pool_).exists(); + +} + +/// @todo: never used in actual fdb-read? +eckit::DataHandle* DaosStore::retrieve(Field& field) const { + + return field.dataHandle(); + +} + +std::unique_ptr DaosStore::archive(const Key &key, const void *data, eckit::Length length) { + + /// @note: performed RPCs: + /// - open pool if not cached (daos_pool_connect) -- always skipped as it is cached after selectDatabase. + /// If the cat backend is toc, then it is performed but only on first write. + /// - check if container exists if not cached (daos_cont_open) -- always skipped as it is cached after selectDatabase. + /// If the cat backend is toc, then it is performed but only on first write. + /// - allocate oid (daos_cont_alloc_oids) -- skipped most of the times as oids per alloc is set to 100 + fdb5::DaosArrayName n = fdb5::DaosName(pool_, db_str_).createArrayName(OC_S1, false); // TODO: pass oclass from config + + std::unique_ptr h(n.dataHandle()); + + /// @note: performed RPCs: + /// - open pool if not cached (daos_pool_connect) -- always skipped, opened above + /// - check if container exists if not cached/open (daos_cont_open) -- always skipped, cached/open above + /// - generate array oid (daos_obj_generate_oid) -- always skipped, already generated above + /// - open container if not open (daos_cont_open) -- always skipped + /// - generate oid again (daos_obj_generate_oid) -- always skipped + /// - create (daos_array_create) -- always performed + /// - open (daos_array_open) -- always skipped, create already opens + h->openForWrite(length); + eckit::AutoClose closer(*h); + + /// @note: performed RPCs: + /// - write (daos_array_write) -- always performed + h->write(data, length); + + return std::make_unique(n.URI(), 0, length, fdb5::Key(nullptr, true)); + + /// @note: performed RPCs: + /// - close (daos_array_close here) -- always performed + +} + +void DaosStore::flush() {} + +void DaosStore::remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const { + + fdb5::DaosName n{uri}; + + ASSERT(n.hasContainerName()); + ASSERT(n.poolName() == pool_); + ASSERT(n.containerName() == db_str_); + + if (n.hasOID()) { + ASSERT(n.OID().otype() == DAOS_OT_ARRAY_BYTE); + logVerbose << "destroy array: "; + } else { + logVerbose << "destroy container: "; + } + + logAlways << n.asString() << std::endl; + if (doit) n.destroy(); + +} + +void DaosStore::print(std::ostream &out) const { + + out << "DaosStore(" << pool_ << ")"; + +} + +static StoreBuilder builder("daos"); + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosStore.h b/src/fdb5/daos/DaosStore.h new file mode 100644 index 000000000..027d03a38 --- /dev/null +++ b/src/fdb5/daos/DaosStore.h @@ -0,0 +1,66 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Nov 2022 + +#pragma once + +#include "fdb5/database/Store.h" +#include "fdb5/rules/Schema.h" + +#include "fdb5/daos/DaosCommon.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosStore : public Store, public DaosCommon { + +public: // methods + + DaosStore(const Schema& schema, const Key& key, const Config& config); + + ~DaosStore() override {} + + eckit::URI uri() const override; + bool uriBelongs(const eckit::URI&) const override; + bool uriExists(const eckit::URI&) const override; + std::vector collocatedDataURIs() const override; + std::set asCollocatedDataURIs(const std::vector&) const override; + + bool open() override { return true; } + void flush() override; + void close() override {}; + + void checkUID() const override { /* nothing to do */ } + +protected: // methods + + std::string type() const override { return "daos"; } + + bool exists() const override; + + eckit::DataHandle* retrieve(Field& field) const override; + std::unique_ptr archive(const Key &key, const void *data, eckit::Length length) override; + + void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override; + + void print(std::ostream &out) const override; + +private: // members + + std::string db_str_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosWipeVisitor.cc b/src/fdb5/daos/DaosWipeVisitor.cc new file mode 100644 index 000000000..7cadab946 --- /dev/null +++ b/src/fdb5/daos/DaosWipeVisitor.cc @@ -0,0 +1,448 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/log/Log.h" +#include "eckit/serialisation/MemoryStream.h" + +#include "fdb5/daos/DaosWipeVisitor.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosSession.h" + +#include + +using namespace eckit; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DaosWipeVisitor::DaosWipeVisitor(const DaosCatalogue& catalogue, + const Store& store, + const metkit::mars::MarsRequest& request, + std::ostream& out, + bool doit, + bool porcelain, + bool unsafeWipeAll) : + WipeVisitor(request, out, doit, porcelain, unsafeWipeAll), + catalogue_(catalogue), + store_(store), + dbKvName_("") {} + +DaosWipeVisitor::~DaosWipeVisitor() {} + +bool DaosWipeVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) { + + // Overall checks + + ASSERT(&catalogue_ == &catalogue); + ASSERT(catalogue.enabled(ControlIdentifier::Wipe)); + WipeVisitor::visitDatabase(catalogue, store); + + // Check that we are in a clean state (i.e. we only visit one DB). + + ASSERT(indexNames_.empty()); + ASSERT(axisNames_.empty()); + ASSERT(safeKvNames_.empty()); + + ASSERT(storeURIs_.empty()); + ASSERT(safeStoreURIs_.empty()); + + ASSERT(!dbKvName_.poolName().size()); + + // Having selected a DB, construct the residual request. This is the request that is used for + // matching Index(es) -- which is relevant if there is subselection of the DB. + + indexRequest_ = request_; + for (const auto& kv : catalogue.key()) { + indexRequest_.unsetValues(kv.first); + } + + return true; // Explore contained indexes +} + +bool DaosWipeVisitor::visitIndex(const Index& index) { + + fdb5::DaosKeyValueName location{index.location().uri()}; + const fdb5::DaosKeyValueName& db_kv = catalogue_.dbKeyValue(); + + // Is this index matched by the supplied request? + // n.b. If the request is over-specified (i.e. below the index level), nothing will be removed + + bool include = index.key().match(indexRequest_); + + // If we have cross fdb-mounted another DB, ensure we can't delete another DBs data. + if (!(location.containerName() == db_kv.containerName() && + location.poolName() == db_kv.poolName())) { + include = false; + } + // ASSERT(location.dirName().sameAs(basePath) || !include); + + // Add the axis and index paths to be removed. + + std::set axes; + fdb5::DaosSession s{}; + fdb5::DaosKeyValue index_kv{s, location}; /// @todo: asserts that kv exists + uint64_t size = index_kv.size("axes"); + if (size > 0) { + std::vector axes_data(size); + index_kv.get("axes", &axes_data[0], size); + std::vector axis_names; + eckit::Tokenizer parse(","); + parse(std::string(axes_data.begin(), axes_data.end()), axis_names); + std::string idx{index.key().valuesToString()}; + for (const auto& name : axis_names) { + /// @todo: take oclass from config + fdb5::DaosKeyValueOID oid(idx + std::string{"."} + name, OC_S1); + fdb5::DaosKeyValueName nkv(location.poolName(), location.containerName(), oid); + axes.insert(nkv); + } + } + + if (include) { + indexNames_.insert(location); + axisNames_.insert(axes.begin(), axes.end()); + } else { + safeKvNames_.insert(location); + safeKvNames_.insert(axes.begin(), axes.end()); + } + + // Enumerate data files. + + /// @note: although a daos index will point to only one daos store container if using a daos store, + /// a daos index can point to multiple posix store files (one per IO server process) if using a posix store. + std::vector indexDataURIs(index.dataURIs()); + for (const eckit::URI& uri : store_.asCollocatedDataURIs(indexDataURIs)) { + if (include) { + if (!store_.uriBelongs(uri)) { + Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." << std::endl; + Log::error() << "Configured Store URI: " << store_.uri().asString() << std::endl; + Log::error() << "Pointed Store unit URI: " << uri.asString() << std::endl; + Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." << std::endl; + throw eckit::SeriousBug{"Index deletion aborted to avoid leaking fields."}; + } + storeURIs_.insert(uri); + } else { + safeStoreURIs_.insert(uri); + } + } + + return true; // Explore contained entries + +} + +void DaosWipeVisitor::ensureSafeURIs() { + + // Very explicitly ensure that we cannot delete anything marked as safe + + for (const auto& p : safeStoreURIs_) + storeURIs_.erase(p); + +} + +void DaosWipeVisitor::calculateResidualURIs() { + + // Remove URIs to non-existant objects. This is reasonable as we may be recovering from a + // previous failed, partial wipe. As such, referenced objects may not exist any more. + + /// @note: indexNames_ always exist at this point, as their existence is ensured in DaosCatalogue::indexes + /// @note: axisNames_ existence cannot be verified as kvs "always exist" in DAOS + + // Consider the total sets of URIs in the DB + + std::set deleteKvNames; + deleteKvNames.insert(indexNames_.begin(), indexNames_.end()); + deleteKvNames.insert(axisNames_.begin(), axisNames_.end()); + if (dbKvName_.poolName().size()) deleteKvNames.insert(dbKvName_.URI()); + + std::set allKvNames; + /// @note: given a database container, list DB kv, index kvs and axis kvs + const fdb5::DaosKeyValueName& db_kv = catalogue_.dbKeyValue(); + fdb5::DaosName db_cont{db_kv.poolName(), db_kv.containerName()}; + for (const auto& oid : db_cont.listOIDs()) { + if (oid.otype() == DAOS_OT_KV_HASHED) { + allKvNames.insert(fdb5::DaosKeyValueName{db_cont.poolName(), db_cont.containerName(), oid}); + } else if (oid.otype() != DAOS_OT_ARRAY_BYTE) { + throw SeriousBug("Found non-KV non-ByteArray objects in DB container " + db_cont.URI().asString()); + } + } + + ASSERT(residualKvNames_.empty()); + + if (!(deleteKvNames == allKvNames)) { + + // First we check if there are names marked to delete that don't exist. This is an error + + std::set names; + std::set_difference(deleteKvNames.begin(), deleteKvNames.end(), + allKvNames.begin(), allKvNames.end(), + std::inserter(names, names.begin())); + + if (!names.empty()) { + Log::error() << "DaosKeyValueNames not in existing names set:" << std::endl; + for (const auto& n : names) { + Log::error() << " - " << n.URI() << std::endl; + } + throw SeriousBug("KV names to delete in deleteKvNames should should be in existing name set. Are multiple wipe commands running simultaneously?", Here()); + } + + std::set_difference(allKvNames.begin(), allKvNames.end(), + deleteKvNames.begin(), deleteKvNames.end(), + std::inserter(residualKvNames_, residualKvNames_.begin())); + } + + // repeat the algorithm for store units (store files or containers) + + for (std::set* uriset : {&storeURIs_}) { + for (std::set::iterator it = uriset->begin(); it != uriset->end(); ) { + + if (store_.uriExists(*it)) { + ++it; + } else { + uriset->erase(it++); + } + + } + } + + std::set deleteStoreURIs; + deleteStoreURIs.insert(storeURIs_.begin(), storeURIs_.end()); + + std::vector allStoreURIsVector; + for (const auto& u : store_.collocatedDataURIs()) { + allStoreURIsVector.push_back(u); + } + std::set allStoreURIs(allStoreURIsVector.begin(), allStoreURIsVector.end()); + + ASSERT(residualStoreURIs_.empty()); + + if (!(storeURIs_ == allStoreURIs)) { + + // First we check if there are URIs marked to delete that don't exist. This is an error + + std::set uris; + std::set_difference(storeURIs_.begin(), storeURIs_.end(), + allStoreURIs.begin(), allStoreURIs.end(), + std::inserter(uris, uris.begin())); + + if (!uris.empty()) { + Log::error() << "Store units (store paths or containers) not in existing paths set:" << std::endl; + for (const auto& u : uris) { + Log::error() << " - " << u << std::endl; + } + throw SeriousBug("Store unit to delete should be in existing URI set. Are multiple wipe commands running simultaneously?", Here()); + } + + std::set_difference(allStoreURIs.begin(), allStoreURIs.end(), + storeURIs_.begin(), storeURIs_.end(), + std::inserter(residualStoreURIs_, residualStoreURIs_.begin())); + + } + +} + +bool DaosWipeVisitor::anythingToWipe() const { + return (!indexNames_.empty() || !axisNames_.empty() || !storeURIs_.empty() || dbKvName_.poolName().size()); +} + +void DaosWipeVisitor::report(bool wipeAll) { + + ASSERT(anythingToWipe()); + + if (wipeAll) { + + out_ << "DB container to delete:" << std::endl; + const fdb5::DaosKeyValueName& db_kv = catalogue_.dbKeyValue(); + out_ << " " << fdb5::DaosName{db_kv.poolName(), db_kv.containerName()}.URI() << std::endl; + out_ << std::endl; + + out_ << "DB KV to delete:" << std::endl; + out_ << " " << db_kv.URI() << std::endl; + out_ << std::endl; + + if (store_.type() != "daos") { + out_ << "Store URI to delete:" << std::endl; + out_ << " " << store_.uri() << std::endl; + out_ << std::endl; + } + + } else { + + out_ << "DB container to delete:" << std::endl; + out_ << " - NONE -" << std::endl; + out_ << std::endl; + out_ << "DB KV to delete:" << std::endl; + out_ << " - NONE -" << std::endl; + out_ << std::endl; + if (store_.type() != "daos") { + out_ << "Store URI to delete:" << std::endl; + out_ << " - NONE -" << std::endl; + out_ << std::endl; + } + + } + + out_ << "Index KVs to delete: " << std::endl; + if (indexNames_.empty()) out_ << " - NONE -" << std::endl; + for (const auto& n : indexNames_) { + out_ << " " << n.URI() << std::endl; + } + out_ << std::endl; + + out_ << "Axis KVs to delete: " << std::endl; + if (axisNames_.empty()) out_ << " - NONE -" << std::endl; + for (const auto& n : axisNames_) { + out_ << " " << n.URI() << std::endl; + } + out_ << std::endl; + + out_ << "Store units (store files or arrays) to delete: " << std::endl; + if (storeURIs_.empty()) out_ << " - NONE -" << std::endl; + for (const auto& f : storeURIs_) { + out_ << " " << f << std::endl; + } + out_ << std::endl; + +} + +void DaosWipeVisitor::wipe(bool wipeAll) { + + ASSERT(anythingToWipe()); + + std::ostream& logAlways(out_); + std::ostream& logVerbose(porcelain_ ? Log::debug() : out_); + + // Now we want to do the actual deletion + // n.b. We delete carefully in a order such that we can always access the DB by what is left + + /// @note: only need to remove residual store paths if the store is not daos. If store is daos, + /// then the entire container will be removed together with residual paths and + /// residual kv paths if wipeAll is true + if (store_.type() != "daos") { + for (const eckit::URI& uri : residualStoreURIs_) { + if (store_.uriExists(uri)) { + store_.remove(uri, logAlways, logVerbose, doit_); + } + } + } + + if (!wipeAll || (wipeAll && store_.type() != "daos")) { + for (const eckit::URI& uri : storeURIs_) { + store_.remove(uri, logAlways, logVerbose, doit_); + } + } + if (!wipeAll) { + for (const fdb5::DaosKeyValueName& name : axisNames_) { + if (name.exists()) { + catalogue_.remove(name, logAlways, logVerbose, doit_); + } + } + for (const fdb5::DaosKeyValueName& name : indexNames_) { + if (name.exists()) { + + fdb5::DaosSession s{}; + fdb5::DaosKeyValue index_kv{s, name}; + std::vector data; + eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index kv"); + fdb5::Key key(ms); + std::string idx{key.valuesToString()}; + + catalogue_.remove(name, logAlways, logVerbose, doit_); + + fdb5::DaosKeyValue db_kv{s, catalogue_.dbKeyValue()}; + if (doit_) db_kv.remove(idx); + + } else { + NOTIMP; + /// iterate catalogue kv to clean dangling reference + /// expensive + } + } + } + + if (wipeAll) { + + if (store_.type() != "daos") + /// @todo: if the store is holding catalogue information (e.g. index files) it + /// should not be removed + store_.remove(store_.uri(), logAlways, logVerbose, doit_); + + const fdb5::DaosKeyValueName& db_kv = catalogue_.dbKeyValue(); + const fdb5::DaosKeyValueName& root_kv = catalogue_.rootKeyValue(); + + fdb5::DaosName db_cont{db_kv.poolName(), db_kv.containerName()}; + + if (db_cont.exists() && doit_) db_cont.destroy(); + + std::string db_key = db_kv.containerName(); + fdb5::DaosSession s{}; + fdb5::DaosKeyValue root{s, root_kv}; + if (root.has(db_key) && doit_) { + root.remove(db_key); + } + } + +} + + +void DaosWipeVisitor::catalogueComplete(const Catalogue& catalogue) { + + WipeVisitor::catalogueComplete(catalogue); + + // We wipe everything if there is nothingn within safe paths - i.e. there is + // no data that wasn't matched by the request (either because all DB indexes were matched + // or because the entire DB was matched) + + bool wipeAll = safeKvNames_.empty() && safeStoreURIs_.empty(); + + dbKvName_ = fdb5::DaosName(""); + if (wipeAll) { + const fdb5::DaosKeyValueName& n = catalogue_.dbKeyValue(); + dbKvName_ = fdb5::DaosName(n.poolName(), n.containerName(), n.OID()); + } + + ensureSafeURIs(); + + if (anythingToWipe()) { + + if (wipeAll) calculateResidualURIs(); + + if (!porcelain_) report(wipeAll); + + // This is here as it needs to run whatever combination of doit/porcelain/... + if (wipeAll && !residualKvNames_.empty()) { + + out_ << "Unexpected KVs present in DB container: " << std::endl; + for (const auto& n : residualKvNames_) out_ << " " << n.URI() << std::endl; + out_ << std::endl; + + } + if (wipeAll && !residualStoreURIs_.empty()) { + + out_ << "Unexpected store units (store files or arrays) present in store: " << std::endl; + for (const auto& u : residualStoreURIs_) out_ << " " << store_.type() << "://" << u << std::endl; + out_ << std::endl; + + } + if (wipeAll && (!residualKvNames_.empty() || !residualStoreURIs_.empty())) { + if (!unsafeWipeAll_) { + out_ << "Full wipe will not proceed without --unsafe-wipe-all" << std::endl; + if (doit_) + throw Exception("Cannot fully wipe unclean Daos DB", Here()); + } + } + + if (doit_ || porcelain_) wipe(wipeAll); + } +} + + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/DaosWipeVisitor.h b/src/fdb5/daos/DaosWipeVisitor.h new file mode 100644 index 000000000..dbaf22f34 --- /dev/null +++ b/src/fdb5/daos/DaosWipeVisitor.h @@ -0,0 +1,74 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date May 2023 + +#pragma once + +#include "fdb5/database/WipeVisitor.h" +#include "fdb5/daos/DaosCatalogue.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DaosWipeVisitor : public WipeVisitor { + +public: + + DaosWipeVisitor(const DaosCatalogue& catalogue, + const Store& store, + const metkit::mars::MarsRequest& request, + std::ostream& out, + bool doit, + bool porcelain, + bool unsafeWipeAll); + ~DaosWipeVisitor() override; + +private: // methods + + bool visitDatabase(const Catalogue& catalogue, const Store& store) override; + bool visitIndex(const Index& index) override; + void catalogueComplete(const Catalogue& catalogue) override; + + void ensureSafeURIs(); + void calculateResidualURIs(); + + bool anythingToWipe() const; + + void report(bool wipeAll); + void wipe(bool wipeAll); + +private: // members + + // What are the parameters of the wipe operation + const DaosCatalogue& catalogue_; + const Store& store_; + + metkit::mars::MarsRequest indexRequest_; + + fdb5::DaosName dbKvName_; + + std::set indexNames_; + std::set axisNames_; + std::set safeKvNames_; + + std::set storeURIs_; + std::set safeStoreURIs_; + + std::set residualKvNames_; + std::set residualStoreURIs_; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/UUID.cc b/src/fdb5/daos/UUID.cc new file mode 100644 index 000000000..bf72f4307 --- /dev/null +++ b/src/fdb5/daos/UUID.cc @@ -0,0 +1,28 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/daos/UUID.h" + +#include "eckit/exception/Exceptions.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +UUID::UUID(const std::string& uuid) { + + if (uuid_parse(uuid.c_str(), internal) != 0) + throw eckit::BadParameter("The provided string is not a uuid."); + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/daos/UUID.h b/src/fdb5/daos/UUID.h new file mode 100644 index 000000000..0e30f5d42 --- /dev/null +++ b/src/fdb5/daos/UUID.h @@ -0,0 +1,40 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date May 2024 + +#pragma once + +#include + +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class UUID { + +public: // methods + + UUID() : internal{0} {} + + UUID(const std::string&); + +public: // members + + uuid_t internal; + +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/database/Catalogue.cc b/src/fdb5/database/Catalogue.cc index 2b4572f38..9bd27c9a7 100644 --- a/src/fdb5/database/Catalogue.cc +++ b/src/fdb5/database/Catalogue.cc @@ -25,13 +25,29 @@ namespace fdb5 { std::unique_ptr Catalogue::buildStore() { - if (buildByKey_) - return StoreFactory::instance().build(schema(), key(), config_); - else { - std::string name = config_.getString("store", "file"); + return StoreFactory::instance().build(schema(), key(), config_); +} + +void Catalogue::visitEntries(EntryVisitor& visitor, const Store& store, bool sorted) { + + std::vector all = indexes(sorted); + + // Allow the visitor to selectively reject this DB. + if (visitor.visitDatabase(*this, store)) { + if (visitor.visitIndexes()) { + for (const Index& idx : all) { + if (visitor.visitEntries()) { + idx.entries(visitor); // contains visitIndex + } else { + visitor.visitIndex(idx); + } + } + } - return StoreFactory::instance().build(schema(), eckit::URI(name, uri()), config_); } + + visitor.catalogueComplete(*this); + } bool Catalogue::enabled(const ControlIdentifier& controlIdentifier) const { diff --git a/src/fdb5/database/Catalogue.h b/src/fdb5/database/Catalogue.h index 733abb5d5..1d7be79fc 100644 --- a/src/fdb5/database/Catalogue.h +++ b/src/fdb5/database/Catalogue.h @@ -44,7 +44,7 @@ class Catalogue { public: Catalogue(const Key& key, ControlIdentifiers controlIdentifiers, const fdb5::Config& config) - : dbKey_(key), config_(config), controlIdentifiers_(controlIdentifiers), buildByKey_(!key.empty()) {} + : dbKey_(key), config_(config), controlIdentifiers_(controlIdentifiers) {} virtual ~Catalogue() {} @@ -60,7 +60,7 @@ class Catalogue { virtual std::vector metadataPaths() const = 0; - virtual void visitEntries(EntryVisitor& visitor, const Store& store, bool sorted = false) = 0; + virtual void visitEntries(EntryVisitor& visitor, const Store& store, bool sorted = false); virtual void hideContents() { NOTIMP; } @@ -108,10 +108,6 @@ class Catalogue { Config config_; ControlIdentifiers controlIdentifiers_; - -private: // members - - bool buildByKey_ = false; }; class CatalogueReader { diff --git a/src/fdb5/database/DatabaseNotFoundException.cc b/src/fdb5/database/DatabaseNotFoundException.cc new file mode 100644 index 000000000..6be6a5709 --- /dev/null +++ b/src/fdb5/database/DatabaseNotFoundException.cc @@ -0,0 +1,24 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/database/DatabaseNotFoundException.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +DatabaseNotFoundException::DatabaseNotFoundException(const std::string& w) : Exception(w) {} + +DatabaseNotFoundException::DatabaseNotFoundException(const std::string& w, const eckit::CodeLocation& l) : + Exception(w, l) {} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/database/DatabaseNotFoundException.h b/src/fdb5/database/DatabaseNotFoundException.h new file mode 100644 index 000000000..66d40b280 --- /dev/null +++ b/src/fdb5/database/DatabaseNotFoundException.h @@ -0,0 +1,30 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date May 2024 + +#pragma once + +#include "eckit/exception/Exceptions.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class DatabaseNotFoundException : public eckit::Exception { +public: + DatabaseNotFoundException(const std::string&); + DatabaseNotFoundException(const std::string&, const eckit::CodeLocation&); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/database/Engine.h b/src/fdb5/database/Engine.h index b9560d50e..eb3663f16 100644 --- a/src/fdb5/database/Engine.h +++ b/src/fdb5/database/Engine.h @@ -25,6 +25,8 @@ #include "eckit/memory/NonCopyable.h" #include "eckit/filesystem/URI.h" +#include "fdb5/database/DB.h" + namespace fdb5 { @@ -48,7 +50,7 @@ class Engine : private eckit::NonCopyable { virtual std::string dbType() const = 0; /// @returns if an Engine is capable of opening this path - virtual bool canHandle(const eckit::URI& uri) const = 0; + virtual bool canHandle(const eckit::URI& uri, const Config&) const = 0; /// Uniquely selects a location where the Key will be put or already exists virtual eckit::URI location(const Key &key, const Config& config) const = 0; diff --git a/src/fdb5/database/EntryVisitMechanism.cc b/src/fdb5/database/EntryVisitMechanism.cc index b57efe294..1d4196de7 100644 --- a/src/fdb5/database/EntryVisitMechanism.cc +++ b/src/fdb5/database/EntryVisitMechanism.cc @@ -14,6 +14,7 @@ #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/database/Manager.h" +#include "fdb5/database/Engine.h" #include "fdb5/LibFdb5.h" #include "fdb5/rules/Schema.h" @@ -90,28 +91,35 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis try { - std::vector uris(Manager(dbConfig_).visitableLocations(request.request(), request.all())); + fdb5::Manager mg{dbConfig_}; + std::vector uris(mg.visitableLocations(request.request(), request.all())); // n.b. it is not an error if nothing is found (especially in a sub-fdb). // And do the visitation for (URI uri : uris) { + /// @note: the schema of a URI returned by visitableLocations + /// matches the corresponding Engine type name + // fdb5::Engine& ng = fdb5::Engine::backend(uri.scheme()); - PathName path(uri.path()); - if (path.exists()) { - if (!path.isDir()) - path = path.dirName(); - path = path.realName(); + std::unique_ptr db; - LOG_DEBUG_LIB(LibFdb5) << "FDB processing Path " << path << std::endl; + try { + + db = DB::buildReader(uri, dbConfig_); - std::unique_ptr db = DB::buildReader(eckit::URI(uri.scheme(), path), dbConfig_); - ASSERT(db->open()); - eckit::AutoCloser closer(*db); + } catch (fdb5::DatabaseNotFoundException& e) { + + visitor.onDatabaseNotFound(e); - db->visitEntries(visitor, false); } + + ASSERT(db->open()); + eckit::AutoCloser closer(*db); + + db->visitEntries(visitor, false); + } } catch (eckit::UserError&) { @@ -121,10 +129,6 @@ void EntryVisitMechanism::visit(const FDBToolRequest& request, EntryVisitor& vis if (fail_) throw; } - - - - } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/fdb5/database/EntryVisitMechanism.h b/src/fdb5/database/EntryVisitMechanism.h index 441818ba8..1c366e441 100644 --- a/src/fdb5/database/EntryVisitMechanism.h +++ b/src/fdb5/database/EntryVisitMechanism.h @@ -18,6 +18,7 @@ #include "fdb5/config/Config.h" #include "fdb5/database/Field.h" +#include "fdb5/database/DatabaseNotFoundException.h" namespace fdb5 { @@ -45,6 +46,8 @@ class EntryVisitor : public eckit::NonCopyable { virtual void catalogueComplete(const Catalogue& catalogue); virtual void visitDatum(const Field& field, const std::string& keyFingerprint); + virtual void onDatabaseNotFound(const fdb5::DatabaseNotFoundException& e) {} + time_t indexTimestamp() const; private: // methods diff --git a/src/fdb5/database/Index.cc b/src/fdb5/database/Index.cc index 613e919eb..4dca73f9c 100755 --- a/src/fdb5/database/Index.cc +++ b/src/fdb5/database/Index.cc @@ -158,7 +158,6 @@ const IndexAxis &IndexBase::axes() const { return axes_; } - //---------------------------------------------------------------------------------------------------------------------- @@ -240,17 +239,6 @@ Index::Index(const Index& s) : content_(s.content_), null_(s.null_) { content_->attach(); } -/*const std::vector Index::dataPaths() const { - std::vector uris = dataUris(); - std::vector paths; - paths.reserve(uris.size()); - - for (eckit::URI& uri: uris) { - paths.emplace_back(uri.path()); - } - return paths; -}*/ - Index& Index::operator=(const Index& s) { content_->detach(); content_ = s.content_; diff --git a/src/fdb5/database/Index.h b/src/fdb5/database/Index.h index 5c9a53c57..c6aec0a15 100755 --- a/src/fdb5/database/Index.h +++ b/src/fdb5/database/Index.h @@ -60,7 +60,7 @@ class IndexBase : public eckit::Counted { virtual const IndexLocation& location() const = 0; - virtual const std::vector dataPaths() const { NOTIMP; } + virtual const std::vector dataURIs() const { NOTIMP; } virtual bool dirty() const = 0; @@ -142,7 +142,7 @@ class Index { const IndexLocation& location() const { return content_->location(); } - const std::vector dataPaths() const { return content_->dataPaths(); } + const std::vector dataURIs() const { return content_->dataURIs(); } bool dirty() const { return content_->dirty(); } diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc index 32dee0583..ff7d5e84c 100755 --- a/src/fdb5/database/IndexAxis.cc +++ b/src/fdb5/database/IndexAxis.cc @@ -275,6 +275,22 @@ void IndexAxis::insert(const Key &key) { } } +/// @note: this method inserts key-value pairs into an axis in memory. +/// Intended for importing axis information from storage in the DAOS backend. +/// Input values are required to be cannoicalised. +void IndexAxis::insert(const std::string& axis, const std::vector& values) { + ASSERT(!readOnly_); + + std::shared_ptr >& axis_set = axis_[axis]; + + if (!axis_set) + axis_set.reset(new eckit::DenseSet()); + + for (const auto& value : values) axis_set->insert(value); + + dirty_ = true; + +} bool IndexAxis::dirty() const { return dirty_; diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h index e4fc4aaac..072ecc77a 100644 --- a/src/fdb5/database/IndexAxis.h +++ b/src/fdb5/database/IndexAxis.h @@ -55,6 +55,8 @@ class IndexAxis : private eckit::NonCopyable { bool operator!=(const IndexAxis& rhs) const; void insert(const Key &key); + /// @note: the values are required to be cannonicalised + void insert(const std::string& axis, const std::vector& values); void encode(eckit::Stream &s, const int version) const; static int currentVersion() { return 3; } diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc index 93828217a..d1e8d089a 100644 --- a/src/fdb5/database/Key.cc +++ b/src/fdb5/database/Key.cc @@ -26,9 +26,9 @@ namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -Key::Key(const std::shared_ptr reg) : +Key::Key(const std::shared_ptr reg, bool canonical) : keys_(), - registry_(reg), canonical_(false) {} + registry_(reg), canonical_(canonical) {} Key::Key(const std::string &s, const Rule *rule) : keys_(), diff --git a/src/fdb5/database/Key.h b/src/fdb5/database/Key.h index 76672e3f4..11b0c9a7c 100644 --- a/src/fdb5/database/Key.h +++ b/src/fdb5/database/Key.h @@ -46,7 +46,7 @@ class Key { public: // methods - explicit Key(const std::shared_ptr reg = nullptr); + explicit Key(const std::shared_ptr reg = nullptr, bool canonical = false); explicit Key(eckit::Stream &, const std::shared_ptr reg = nullptr); explicit Key(const std::string &keys, const Rule* rule); explicit Key(const eckit::StringDict &keys, const std::shared_ptr reg=nullptr); diff --git a/src/fdb5/database/Manager.cc b/src/fdb5/database/Manager.cc index fc5d61eae..a9b033f19 100644 --- a/src/fdb5/database/Manager.cc +++ b/src/fdb5/database/Manager.cc @@ -259,7 +259,7 @@ std::string Manager::engine(const URI& uri) for(std::vector::const_iterator i = engines.begin(); i != engines.end(); ++i) { ASSERT(*i); const Engine& e = **i; - if(e.canHandle(uri)) { + if(e.canHandle(uri, config_)) { return e.dbType(); } } @@ -310,6 +310,7 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ } else { p = Engine::backend(*i).visitableLocations(rq, config_); } + r.insert(r.end(), p.begin(), p.end()); } diff --git a/src/fdb5/database/Store.cc b/src/fdb5/database/Store.cc index 570e7ba68..6c52b85cf 100644 --- a/src/fdb5/database/Store.cc +++ b/src/fdb5/database/Store.cc @@ -92,26 +92,6 @@ std::unique_ptr StoreFactory::build(const Schema& schema, const Key& key, return (*j).second->make(schema, key, config); } -std::unique_ptr StoreFactory::build(const Schema& schema, const eckit::URI& uri, const Config& config) { - std::string name = uri.scheme(); - std::string nameLowercase = eckit::StringTools::lower(name); - - eckit::AutoLock lock(mutex_); - auto j = builders_.find(nameLowercase); - - LOG_DEBUG_LIB(LibFdb5) << "Looking for StoreBuilder [" << nameLowercase << "]" << std::endl; - - if (j == builders_.end()) { - eckit::Log::error() << "No StoreBuilder for [" << nameLowercase << "]" << std::endl; - eckit::Log::error() << "StoreBuilders are:" << std::endl; - for (j = builders_.begin(); j != builders_.end(); ++j) - eckit::Log::error() << " " << (*j).first << std::endl; - throw eckit::SeriousBug(std::string("No StoreBuilder called ") + nameLowercase); - } - - return (*j).second->make(schema, uri, config); -} - //---------------------------------------------------------------------------------------------------------------------- StoreBuilderBase::StoreBuilderBase(const std::string& name) : name_(name) { diff --git a/src/fdb5/database/Store.h b/src/fdb5/database/Store.h index 268359b39..9d271e8d2 100644 --- a/src/fdb5/database/Store.h +++ b/src/fdb5/database/Store.h @@ -57,6 +57,10 @@ class Store { virtual void remove(const Key& key) const { NOTIMP; } virtual eckit::URI uri() const = 0; + virtual bool uriBelongs(const eckit::URI&) const = 0; + virtual bool uriExists(const eckit::URI& uri) const = 0; + virtual std::vector collocatedDataURIs() const = 0; + virtual std::set asCollocatedDataURIs(const std::vector&) const = 0; protected: // members const Schema& schema_; //<< schema is owned by catalogue which always outlives the store @@ -72,13 +76,11 @@ class StoreBuilderBase { StoreBuilderBase(const std::string&); virtual ~StoreBuilderBase(); virtual std::unique_ptr make(const Schema& schema, const Key& key, const Config& config) = 0; - virtual std::unique_ptr make(const Schema& schema, const eckit::URI& uri, const Config& config) = 0; }; template class StoreBuilder : public StoreBuilderBase { virtual std::unique_ptr make(const Schema& schema, const Key& key, const Config& config) override { return std::unique_ptr(new T(schema, key, config)); } - virtual std::unique_ptr make(const Schema& schema, const eckit::URI& uri, const Config& config) override { return std::unique_ptr(new T(schema, uri, config)); } public: StoreBuilder(const std::string& name) : StoreBuilderBase(name) {} @@ -101,12 +103,6 @@ class StoreFactory { /// @returns store built by specified builder std::unique_ptr build(const Schema& schema, const Key& key, const Config& config); - /// @param schema the schema read by the catalog - /// @param uri search uri - /// @param config the fdb config - /// @returns store built by specified builder - std::unique_ptr build(const Schema& schema, const eckit::URI& uri, const Config& config); - private: StoreFactory(); diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in index 6fec3c5d5..ab6d25a57 100644 --- a/src/fdb5/fdb5_config.h.in +++ b/src/fdb5/fdb5_config.h.in @@ -11,6 +11,9 @@ #cmakedefine fdb5_HAVE_PMEMFDB #cmakedefine fdb5_HAVE_RADOSFDB #cmakedefine fdb5_HAVE_TOCFDB +#cmakedefine fdb5_HAVE_DUMMY_DAOS +#cmakedefine fdb5_HAVE_DAOSFDB +#cmakedefine fdb5_HAVE_DAOS_ADMIN #cmakedefine01 fdb5_HAVE_GRIB #endif // fdb5_fdb5_config_h diff --git a/src/fdb5/pmem/PMemIndex.cc b/src/fdb5/pmem/PMemIndex.cc index 558b63a40..a9496e316 100755 --- a/src/fdb5/pmem/PMemIndex.cc +++ b/src/fdb5/pmem/PMemIndex.cc @@ -141,10 +141,15 @@ std::string PMemIndex::defaulType() { return "PMemIndex"; } -const std::vector PMemIndex::dataPaths() const { +const std::vector PMemIndex::dataURIs() const { // n.b. this lists the pools that _could_ be referenced, not those that // necessarily are. That would need proper enumeration of all contents. - return location_.pool_manager().dataPoolPaths(); + + std::vector result; + for (auto path : location_.pool_manager().dataPoolPaths()) + result.push_back(eckit::URI{"pmem", path}); + + return result; } void PMemIndex::dump(std::ostream& out, const char* indent, bool simple, bool dumpFields) const { diff --git a/src/fdb5/pmem/PMemIndex.h b/src/fdb5/pmem/PMemIndex.h index a7ba32bde..208624979 100755 --- a/src/fdb5/pmem/PMemIndex.h +++ b/src/fdb5/pmem/PMemIndex.h @@ -49,7 +49,7 @@ class PMemIndex : public IndexBase { protected: // methods virtual const IndexLocation& location() const { return location_; } - virtual const std::vector dataPaths() const override; + virtual const std::vector dataURIs() const override; virtual bool dirty() const; diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index f72dd27b4..c2de5df13 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -31,9 +31,6 @@ namespace fdb5 { RadosStore::RadosStore(const Schema& schema, const Key& key, const Config& config) : Store(schema), directory_("mars:"+key.valuesToString()) {} -RadosStore::RadosStore(const Schema& schema, const eckit::URI& uri, const Config& config) : - Store(schema), directory_("mars:"+uri.path().dirName()) {} - eckit::URI RadosStore::uri() const { return URI("rados", directory_); } diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index b35189be6..c188ad95d 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -31,7 +31,6 @@ class RadosStore : public Store { public: // methods RadosStore(const Schema& schema, const Key& key, const Config& config); - RadosStore(const Schema& schema, const eckit::URI& uri, const Config& config); ~RadosStore() override {} diff --git a/src/fdb5/remote/RemoteFieldLocation.cc b/src/fdb5/remote/RemoteFieldLocation.cc index a552f5474..52562fe3e 100644 --- a/src/fdb5/remote/RemoteFieldLocation.cc +++ b/src/fdb5/remote/RemoteFieldLocation.cc @@ -118,4 +118,4 @@ class FdbURIManager : public eckit::URIManager { static FdbURIManager manager_fdb_file("fdb"); } // namespace remote -} // namespace fdb5 +} // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/toc/FieldRef.cc b/src/fdb5/toc/FieldRef.cc index ec5e0900c..db500336b 100644 --- a/src/fdb5/toc/FieldRef.cc +++ b/src/fdb5/toc/FieldRef.cc @@ -14,11 +14,10 @@ #include "eckit/filesystem/URI.h" #include "eckit/serialisation/Stream.h" +#include "fdb5/fdb5_config.h" #include "fdb5/database/Field.h" #include "fdb5/database/UriStore.h" -#include "fdb5/toc/TocFieldLocation.h" - namespace fdb5 { @@ -33,14 +32,11 @@ FieldRefLocation::FieldRefLocation() { FieldRefLocation::FieldRefLocation(UriStore &store, const Field& field) { const FieldLocation& loc = field.location(); - const TocFieldLocation* tocfloc = dynamic_cast(&loc); - if(!tocfloc) { - throw eckit::NotImplemented("Field location is not of TocFieldLocation type -- indexing other locations is not supported", Here()); - } - - uriId_ = store.insert(tocfloc->uri()); - length_ = tocfloc->length(); - offset_ = tocfloc->offset(); + + uriId_ = store.insert(loc.uri()); + length_ = loc.length(); + offset_ = loc.offset(); + } void FieldRefLocation::print(std::ostream &s) const { diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc index 7377bb67a..c3d359b67 100644 --- a/src/fdb5/toc/TocCatalogue.cc +++ b/src/fdb5/toc/TocCatalogue.cc @@ -78,25 +78,6 @@ std::vector TocCatalogue::metadataPaths() const { return paths; } -void TocCatalogue::visitEntries(EntryVisitor& visitor, const Store& store, bool sorted) { - - std::vector all = indexes(sorted); - - // Allow the visitor to selectively reject this DB. - if (visitor.visitDatabase(*this, store)) { - if (visitor.visitIndexes()) { - for (const Index& idx : all) { - if (visitor.visitEntries()) { - idx.entries(visitor); // contains visitIndex - } else { - visitor.visitIndex(idx); - } - } - } - } - visitor.catalogueComplete(*this); -} - void TocCatalogue::loadSchema() { Timer timer("TocCatalogue::loadSchema()", Log::debug()); schema_ = &SchemaRegistry::instance().get(schemaPath()); diff --git a/src/fdb5/toc/TocCatalogue.h b/src/fdb5/toc/TocCatalogue.h index c4bbda046..8a2c79d9f 100644 --- a/src/fdb5/toc/TocCatalogue.h +++ b/src/fdb5/toc/TocCatalogue.h @@ -58,7 +58,6 @@ class TocCatalogue : public Catalogue, public TocHandler { void checkUID() const override; bool exists() const override; - void visitEntries(EntryVisitor& visitor, const Store& store, bool sorted) override; void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override; std::vector metadataPaths() const override; const Schema& schema() const override; diff --git a/src/fdb5/toc/TocCatalogueReader.cc b/src/fdb5/toc/TocCatalogueReader.cc index e49b67b54..ac4b24594 100644 --- a/src/fdb5/toc/TocCatalogueReader.cc +++ b/src/fdb5/toc/TocCatalogueReader.cc @@ -37,6 +37,7 @@ TocCatalogueReader::~TocCatalogueReader() { void TocCatalogueReader::loadIndexesAndRemap() { std::vector remapKeys; + /// @todo: this should throw DatabaseNotFoundException if the toc file is not found std::vector indexes = loadIndexes(false, nullptr, nullptr, &remapKeys); ASSERT(remapKeys.size() == indexes.size()); diff --git a/src/fdb5/toc/TocEngine.cc b/src/fdb5/toc/TocEngine.cc index 4f3d83df6..b50bc5517 100644 --- a/src/fdb5/toc/TocEngine.cc +++ b/src/fdb5/toc/TocEngine.cc @@ -121,7 +121,7 @@ eckit::URI TocEngine::location(const Key& key, const Config& config) const return URI("toc", CatalogueRootManager(config).directory(key).directory_); } -bool TocEngine::canHandle(const eckit::URI& uri) const +bool TocEngine::canHandle(const eckit::URI& uri, const Config& config) const { if (uri.scheme() != "toc") return false; @@ -209,7 +209,7 @@ std::vector TocEngine::databases(const Key& key, TocHandler toc(path, config); if (toc.databaseKey().match(key)) { LOG_DEBUG_LIB(LibFdb5) << " found match with " << path << std::endl; - result.push_back(eckit::URI("toc", path)); + result.push_back(eckit::URI(typeName(), path)); } } catch (eckit::Exception& e) { eckit::Log::error() << "Error loading FDB database from " << path << std::endl; diff --git a/src/fdb5/toc/TocEngine.h b/src/fdb5/toc/TocEngine.h index 8c5af1709..d4d70db24 100644 --- a/src/fdb5/toc/TocEngine.h +++ b/src/fdb5/toc/TocEngine.h @@ -47,7 +47,7 @@ class TocEngine : public fdb5::Engine { virtual eckit::URI location(const Key &key, const Config& config) const override; - virtual bool canHandle(const eckit::URI& path) const override; + virtual bool canHandle(const eckit::URI&, const Config& config) const override; virtual std::vector allLocations(const Key& key, const Config& config) const override; diff --git a/src/fdb5/toc/TocHandler.cc b/src/fdb5/toc/TocHandler.cc index 5834c985d..eddd91117 100644 --- a/src/fdb5/toc/TocHandler.cc +++ b/src/fdb5/toc/TocHandler.cc @@ -1274,8 +1274,8 @@ void TocHandler::enumerateMasked(std::set>& metada std::vector indexes = h.loadIndexes(); for (const auto& i : indexes) { metadata.insert(std::make_pair(i.location().uri(), 0)); - for (const auto& dataPath : i.dataPaths()) { - data.insert(dataPath); + for (const auto& dataURI : i.dataURIs()) { + data.insert(dataURI); } } } @@ -1310,7 +1310,7 @@ void TocHandler::enumerateMasked(std::set>& metada if (maskedEntries_.find(key) != maskedEntries_.end()) { if (absPath.exists()) { Index index(new TocIndex(s, r->header_.serialisationVersion_, directory_, absPath, offset)); - for (const auto& dataPath : index.dataPaths()) data.insert(dataPath); + for (const auto& dataURI : index.dataURIs()) data.insert(dataURI); } } } diff --git a/src/fdb5/toc/TocIndex.cc b/src/fdb5/toc/TocIndex.cc index 3f6737f8a..1cba36faa 100644 --- a/src/fdb5/toc/TocIndex.cc +++ b/src/fdb5/toc/TocIndex.cc @@ -199,7 +199,7 @@ std::string TocIndex::defaulType() { return BTreeIndex::defaulType(); } -const std::vector TocIndex::dataPaths() const { +const std::vector TocIndex::dataURIs() const { return files_.paths(); } diff --git a/src/fdb5/toc/TocIndex.h b/src/fdb5/toc/TocIndex.h index 6d5874954..ea1347832 100644 --- a/src/fdb5/toc/TocIndex.h +++ b/src/fdb5/toc/TocIndex.h @@ -87,7 +87,7 @@ class TocIndex : private: // methods const IndexLocation& location() const override { return location_; } - const std::vector dataPaths() const override; + const std::vector dataURIs() const override; bool dirty() const override; diff --git a/src/fdb5/toc/TocStore.cc b/src/fdb5/toc/TocStore.cc index d32d4e6b6..e61a981cc 100644 --- a/src/fdb5/toc/TocStore.cc +++ b/src/fdb5/toc/TocStore.cc @@ -37,15 +37,69 @@ namespace fdb5 { TocStore::TocStore(const Schema& schema, const Key& key, const Config& config) : Store(schema), TocCommon(StoreRootManager(config).directory(key).directory_) {} -TocStore::TocStore(const Schema& schema, const eckit::URI& uri, const Config& config) : - Store(schema), TocCommon(uri.path().dirName()) {} - eckit::URI TocStore::uri() const { + return URI("file", directory_); + +} + +bool TocStore::uriBelongs(const eckit::URI& uri) const { + + // TODO: assert uri represents a (not necessarily existing) data file + return ((uri.scheme() == type()) && (uri.path().dirName().sameAs(directory_))); + +} + +bool TocStore::uriExists(const eckit::URI& uri) const { + + ASSERT(uri.scheme() == type()); + eckit::PathName p(uri.path()); + // ensure provided URI is either DB URI or Store file URI + if (!p.sameAs(directory_)) { + ASSERT(p.dirName().sameAs(directory_)); + ASSERT(p.extension() == ".data"); + } + + return p.exists(); + +} + +std::vector TocStore::collocatedDataURIs() const { + + std::vector files; + std::vector dirs; + (directory_).children(files, dirs); + + std::vector res; + for (const auto& f : files) { + if (f.extension() == ".data") { + res.push_back(eckit::URI{type(), f}); + } + } + + return res; + +} + +std::set TocStore::asCollocatedDataURIs(const std::vector& uris) const { + + std::set res; + + for (auto& uri : uris) { + + ASSERT(uri.path().extension() == ".data"); + res.insert(uri); + + } + + return res; + } bool TocStore::exists() const { + return directory_.exists(); + } eckit::DataHandle* TocStore::retrieve(Field& field) const { @@ -65,7 +119,7 @@ std::unique_ptr TocStore::archive(const Key &key, const void *dat ASSERT(len == length); - return std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key())); + return std::unique_ptr(new TocFieldLocation(dataPath, position, length, Key(nullptr, true))); } void TocStore::flush() { @@ -234,7 +288,7 @@ void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& de eckit::PathName destPath = dest.path(); for (const eckit::PathName& root: StoreRootManager(config).canMoveToRoots(key)) { if (root.sameAs(destPath)) { - eckit::PathName src_db = directory_ / key.valuesToString(); + eckit::PathName src_db = directory_; eckit::PathName dest_db = destPath / key.valuesToString(); dest_db.mkdir(); @@ -260,7 +314,7 @@ void TocStore::moveTo(const Key& key, const Config& config, const eckit::URI& de void TocStore::remove(const Key& key) const { - eckit::PathName src_db = directory_ / key.valuesToString(); + eckit::PathName src_db = directory_; DIR* dirp = ::opendir(src_db.asString().c_str()); struct dirent* dp; diff --git a/src/fdb5/toc/TocStore.h b/src/fdb5/toc/TocStore.h index 10500748b..b81c0fb7c 100644 --- a/src/fdb5/toc/TocStore.h +++ b/src/fdb5/toc/TocStore.h @@ -34,11 +34,14 @@ class TocStore : public Store, public TocCommon { public: // methods TocStore(const Schema& schema, const Key& key, const Config& config); - TocStore(const Schema& schema, const eckit::URI& uri, const Config& config); ~TocStore() override {} eckit::URI uri() const override; + bool uriBelongs(const eckit::URI&) const override; + bool uriExists(const eckit::URI&) const override; + std::vector collocatedDataURIs() const override; + std::set asCollocatedDataURIs(const std::vector&) const override; bool open() override { return true; } void flush() override; diff --git a/src/fdb5/toc/TocWipeVisitor.cc b/src/fdb5/toc/TocWipeVisitor.cc index eeba7204d..d9f952eca 100644 --- a/src/fdb5/toc/TocWipeVisitor.cc +++ b/src/fdb5/toc/TocWipeVisitor.cc @@ -24,7 +24,6 @@ #include #include - using namespace eckit; namespace fdb5 { @@ -162,12 +161,19 @@ bool TocWipeVisitor::visitIndex(const Index& index) { // Enumerate data files. - std::vector indexDataPaths(index.dataPaths()); - for (const eckit::URI& uri : indexDataPaths) { - if (include && uri.path().dirName().sameAs(basePath)) { - dataPaths_.insert(uri.path()); + std::vector indexDataPaths(index.dataURIs()); + for (const eckit::URI& uri : store_.asCollocatedDataURIs(indexDataPaths)) { + if (include) { + if (!store_.uriBelongs(uri)) { + Log::error() << "Index to be deleted has pointers to fields that don't belong to the configured store." << std::endl; + Log::error() << "Configured Store URI: " << store_.uri().asString() << std::endl; + Log::error() << "Pointed Store unit URI: " << uri.asString() << std::endl; + Log::error() << "Impossible to delete such fields. Index deletion aborted to avoid leaking fields." << std::endl; + NOTIMP; + } + dataPaths_.insert(eckit::PathName(uri.path())); } else { - safePaths_.insert(uri.path()); + safePaths_.insert(eckit::PathName(uri.path())); } } @@ -192,7 +198,7 @@ void TocWipeVisitor::addMaskedPaths() { } } for (const auto& uri : data) { - if (uri.path().dirName().sameAs(catalogue_.basePath())) dataPaths_.insert(uri.path()); + if (store_.uriBelongs(uri)) dataPaths_.insert(eckit::PathName(uri.path())); } } @@ -232,15 +238,30 @@ void TocWipeVisitor::calculateResidualPaths() { // Remove paths to non-existant files. This is reasonable as we may be recovering from a // previous failed, partial wipe. As such, referenced files may not exist any more. - for (std::set* fileset : {&subtocPaths_, &lockfilePaths_, &indexPaths_, &dataPaths_}) { + for (std::set* fileset : {&subtocPaths_, &lockfilePaths_, &indexPaths_}) { for (std::set::iterator it = fileset->begin(); it != fileset->end(); ) { + if (it->exists()) { ++it; } else { fileset->erase(it++); } + } } + + for (std::set* fileset : {&dataPaths_}) { + for (std::set::iterator it = fileset->begin(); it != fileset->end(); ) { + + if (store_.uriExists(eckit::URI(store_.type(), *it))) { + ++it; + } else { + fileset->erase(it++); + } + + } + } + if (tocPath_.asString().size() && !tocPath_.exists()) tocPath_ = ""; if (schemaPath_.asString().size() && !schemaPath_.exists()) @@ -252,7 +273,8 @@ void TocWipeVisitor::calculateResidualPaths() { deletePaths.insert(subtocPaths_.begin(), subtocPaths_.end()); deletePaths.insert(lockfilePaths_.begin(), lockfilePaths_.end()); deletePaths.insert(indexPaths_.begin(), indexPaths_.end()); - deletePaths.insert(dataPaths_.begin(), dataPaths_.end()); + if (store_.type() == "file") + deletePaths.insert(dataPaths_.begin(), dataPaths_.end()); if (tocPath_.asString().size()) deletePaths.insert(tocPath_); if (schemaPath_.asString().size()) deletePaths.insert(schemaPath_); @@ -273,10 +295,10 @@ void TocWipeVisitor::calculateResidualPaths() { std::inserter(paths, paths.begin())); if (!paths.empty()) { - Log::error() << "Paths not in existing paths set:" << std::endl; - for (const auto& p : paths) { - Log::error() << " - " << p << std::endl; - } + Log::error() << "Paths not in existing paths set:" << std::endl; + for (const auto& p : paths) { + Log::error() << " - " << p << std::endl; + } throw SeriousBug("Path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", Here()); } @@ -284,6 +306,44 @@ void TocWipeVisitor::calculateResidualPaths() { deletePaths.begin(), deletePaths.end(), std::inserter(residualPaths_, residualPaths_.begin())); } + + // if the store uses a backend other than POSIX (file), repeat the algorithm specialized + // for its store units + + if (store_.type() == "file") return; + + std::vector allCollocatedDataURIs(store_.collocatedDataURIs()); + std::vector allDataPathsVector; + for (const auto& u : allCollocatedDataURIs) { + allDataPathsVector.push_back(eckit::PathName(u.path())); + } + + std::set allDataPaths(allDataPathsVector.begin(), allDataPathsVector.end()); + + ASSERT(residualDataPaths_.empty()); + + if (!(dataPaths_ == allDataPaths)) { + + // First we check if there are paths marked to delete that don't exist. This is an error + + std::set paths; + std::set_difference(dataPaths_.begin(), dataPaths_.end(), + allDataPaths.begin(), allDataPaths.end(), + std::inserter(paths, paths.begin())); + + if (!paths.empty()) { + Log::error() << "Store unit paths not in existing paths set:" << std::endl; + for (const auto& p : paths) { + Log::error() << " - " << p << std::endl; + } + throw SeriousBug("Store unit path to delete should be in existing path set. Are multiple wipe commands running simultaneously?", Here()); + } + + std::set_difference(allDataPaths.begin(), allDataPaths.end(), + dataPaths_.begin(), dataPaths_.end(), + std::inserter(residualDataPaths_, residualDataPaths_.begin())); + } + } bool TocWipeVisitor::anythingToWipe() const { @@ -292,7 +352,7 @@ bool TocWipeVisitor::anythingToWipe() const { tocPath_.asString().size() || schemaPath_.asString().size()); } -void TocWipeVisitor::report() { +void TocWipeVisitor::report(bool wipeAll) { ASSERT(anythingToWipe()); @@ -329,6 +389,16 @@ void TocWipeVisitor::report() { } out_ << std::endl; + if (store_.type() != "file") { + out_ << "Store URI to delete:" << std::endl; + if (wipeAll) { + out_ << " " << store_.uri() << std::endl; + } else { + out_ << " - NONE -" << std::endl; + } + out_ << std::endl; + } + out_ << "Protected files (explicitly untouched):" << std::endl; if (safePaths_.empty()) out_ << " - NONE - " << std::endl; for (const auto& f : safePaths_) { @@ -387,6 +457,15 @@ void TocWipeVisitor::wipe(bool wipeAll) { // Now we want to do the actual deletion // n.b. We delete carefully in a order such that we can always access the DB by what is left + + /// @todo: are all these exist checks necessary? + + for (const PathName& path : residualDataPaths_) { + eckit::URI uri(store_.type(), path); + if (store_.uriExists(uri)) { + store_.remove(uri, logAlways, logVerbose, doit_); + } + } for (const PathName& path : residualPaths_) { if (path.exists()) { catalogue_.remove(path, logAlways, logVerbose, doit_); @@ -394,9 +473,17 @@ void TocWipeVisitor::wipe(bool wipeAll) { } for (const PathName& path : dataPaths_) { - store_.remove(eckit::URI(store_.type(), path), logAlways, logVerbose, doit_); + eckit::URI uri(store_.type(), path); + if (store_.uriExists(uri)) { + store_.remove(uri, logAlways, logVerbose, doit_); + } } + if (wipeAll && store_.type() != "file") + /// @todo: if the store is holding catalogue information (e.g. daos KVs) it + /// should not be removed + store_.remove(store_.uri(), logAlways, logVerbose, doit_); + for (const std::set& pathset : {indexPaths_, std::set{schemaPath_}, subtocPaths_, std::set{tocPath_}, lockfilePaths_, @@ -435,7 +522,7 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { if (anythingToWipe()) { if (wipeAll) calculateResidualPaths(); - if (!porcelain_) report(); + if (!porcelain_) report(wipeAll); // This is here as it needs to run whatever combination of doit/porcelain/... if (wipeAll && !residualPaths_.empty()) { @@ -444,6 +531,15 @@ void TocWipeVisitor::catalogueComplete(const Catalogue& catalogue) { for (const auto& p : residualPaths_) out_ << " " << p << std::endl; out_ << std::endl; + } + if (wipeAll && !residualDataPaths_.empty()) { + + out_ << "Unexpected store units present in store: " << std::endl; + for (const auto& p : residualDataPaths_) out_ << " " << store_.type() << "://" << p << std::endl; + out_ << std::endl; + + } + if (wipeAll && (!residualPaths_.empty() || !residualDataPaths_.empty())) { if (!unsafeWipeAll_) { out_ << "Full wipe will not proceed without --unsafe-wipe-all" << std::endl; if (doit_) diff --git a/src/fdb5/toc/TocWipeVisitor.h b/src/fdb5/toc/TocWipeVisitor.h index be1352396..6b1f88c18 100644 --- a/src/fdb5/toc/TocWipeVisitor.h +++ b/src/fdb5/toc/TocWipeVisitor.h @@ -48,7 +48,7 @@ class TocWipeVisitor : public WipeVisitor { bool anythingToWipe() const; - void report(); + void report(bool wipeAll); void wipe(bool wipeAll); private: // members @@ -71,6 +71,7 @@ class TocWipeVisitor : public WipeVisitor { std::set safePaths_; std::set residualPaths_; + std::set residualDataPaths_; std::vector indexesToMask_; }; diff --git a/src/fdb5/tools/FDBTool.cc b/src/fdb5/tools/FDBTool.cc index 878e2f674..4af732172 100644 --- a/src/fdb5/tools/FDBTool.cc +++ b/src/fdb5/tools/FDBTool.cc @@ -45,7 +45,7 @@ void FDBTool::run() { finish(args); } -Config FDBTool::config(const eckit::option::CmdArgs& args) const { +Config FDBTool::config(const eckit::option::CmdArgs& args, const eckit::Configuration& userConfig) const { if (args.has("config")) { std::string config = args.getString("config", ""); @@ -63,10 +63,10 @@ Config FDBTool::config(const eckit::option::CmdArgs& args) const { ss << "Path " << config << " is a directory. Expecting a file"; throw eckit::UserError(ss.str(), Here()); } - return Config::make(configPath); + return Config::make(configPath, userConfig); } - return LibFdb5::instance().defaultConfig(); + return LibFdb5::instance().defaultConfig(userConfig); } void FDBTool::usage(const std::string&) const {} diff --git a/src/fdb5/tools/FDBTool.h b/src/fdb5/tools/FDBTool.h index 14bbc15d1..aa3db4a34 100644 --- a/src/fdb5/tools/FDBTool.h +++ b/src/fdb5/tools/FDBTool.h @@ -46,7 +46,7 @@ class FDBTool : public eckit::Tool { virtual ~FDBTool() override {} virtual void run() override; - Config config(const eckit::option::CmdArgs& args) const; + Config config(const eckit::option::CmdArgs& args, const eckit::Configuration& userConfig = eckit::LocalConfiguration()) const; public: // methods diff --git a/src/fdb5/tools/fdb-hammer.cc b/src/fdb5/tools/fdb-hammer.cc index 6c95cdf0d..bb1757497 100644 --- a/src/fdb5/tools/fdb-hammer.cc +++ b/src/fdb5/tools/fdb-hammer.cc @@ -8,11 +8,14 @@ * does it submit to any jurisdiction. */ +#include +#include #include #include #include "eccodes.h" +#include "eckit/config/Resource.h" #include "eckit/config/LocalConfiguration.h" #include "eckit/io/DataHandle.h" #include "eckit/io/StdFile.h" @@ -25,6 +28,7 @@ #include "fdb5/message/MessageArchiver.h" #include "fdb5/io/HandleGatherer.h" #include "fdb5/tools/FDBTool.h" +#include "fdb5/api/helpers/FDBToolRequest.h" // This list is currently sufficient to get to nparams=200 of levtype=ml,type=fc const std::unordered_set AWKWARD_PARAMS {11, 12, 13, 14, 15, 16, 49, 51, 52, 61, 121, 122, 146, 147, 169, 175, 176, 177, 179, 189, 201, 202}; @@ -33,7 +37,7 @@ const std::unordered_set AWKWARD_PARAMS {11, 12, 13, 14, 15, 16, 49, 51, using namespace eckit; -class FDBWrite : public fdb5::FDBTool { +class FDBHammer : public fdb5::FDBTool { virtual void usage(const std::string &tool) const override; @@ -45,10 +49,11 @@ class FDBWrite : public fdb5::FDBTool { void executeRead(const eckit::option::CmdArgs& args); void executeWrite(const eckit::option::CmdArgs& args); + void executeList(const eckit::option::CmdArgs& args); public: - FDBWrite(int argc, char **argv) : + FDBHammer(int argc, char **argv) : fdb5::FDBTool(argc, argv), verbose_(false) { @@ -56,25 +61,28 @@ class FDBWrite : public fdb5::FDBTool { options_.push_back(new eckit::option::SimpleOption("class", "Reset class on data")); options_.push_back(new eckit::option::SimpleOption("statistics", "Report statistics after run")); options_.push_back(new eckit::option::SimpleOption("read", "Read rather than write the data")); + options_.push_back(new eckit::option::SimpleOption("list", "List rather than write the data")); options_.push_back(new eckit::option::SimpleOption("nsteps", "Number of steps")); options_.push_back(new eckit::option::SimpleOption("nensembles", "Number of ensemble members")); options_.push_back(new eckit::option::SimpleOption("number", "The first ensemble number to use")); options_.push_back(new eckit::option::SimpleOption("nlevels", "Number of levels")); + options_.push_back(new eckit::option::SimpleOption("level", "The first level number to use")); options_.push_back(new eckit::option::SimpleOption("nparams", "Number of parameters")); options_.push_back(new eckit::option::SimpleOption("verbose", "Print verbose output")); + options_.push_back(new eckit::option::SimpleOption("disable-subtocs", "Disable use of subtocs")); } - ~FDBWrite() override {} + ~FDBHammer() override {} private: bool verbose_; }; -void FDBWrite::usage(const std::string &tool) const { - eckit::Log::info() << std::endl << "Usage: " << tool << " [--statistics] [--read] --nsteps= --nensembles= --nlevels= --nparams= --expver= " << std::endl; +void FDBHammer::usage(const std::string &tool) const { + eckit::Log::info() << std::endl << "Usage: " << tool << " [--statistics] [--read] [--list] --nsteps= --nensembles= --nlevels= --nparams= --expver= " << std::endl; fdb5::FDBTool::usage(tool); } -void FDBWrite::init(const eckit::option::CmdArgs& args) +void FDBHammer::init(const eckit::option::CmdArgs& args) { FDBTool::init(args); @@ -87,16 +95,18 @@ void FDBWrite::init(const eckit::option::CmdArgs& args) verbose_ = args.getBool("verbose", false); } -void FDBWrite::execute(const eckit::option::CmdArgs &args) { +void FDBHammer::execute(const eckit::option::CmdArgs &args) { if (args.getBool("read", false)) { executeRead(args); + } else if (args.getBool("list", false)) { + executeList(args); } else { executeWrite(args); } } -void FDBWrite::executeWrite(const eckit::option::CmdArgs &args) { +void FDBHammer::executeWrite(const eckit::option::CmdArgs &args) { eckit::AutoStdFile fin(args(0)); @@ -108,13 +118,17 @@ void FDBWrite::executeWrite(const eckit::option::CmdArgs &args) { size_t nensembles = args.getLong("nensembles", 1); size_t nlevels = args.getLong("nlevels"); size_t nparams = args.getLong("nparams"); - size_t number = args.getLong("number", 1); + size_t number = args.getLong("number", 1); + size_t level = args.getLong("level", 1); const char* buffer = nullptr; size_t size = 0; - fdb5::MessageArchiver archiver(fdb5::Key(), false, verbose_, args); + eckit::LocalConfiguration userConfig{}; + if (!args.has("disable-subtocs")) userConfig.set("useSubToc", true); + + fdb5::MessageArchiver archiver(fdb5::Key(), false, verbose_, config(args, userConfig)); std::string expver = args.getString("expver"); size = expver.length(); @@ -123,6 +137,7 @@ void FDBWrite::executeWrite(const eckit::option::CmdArgs &args) { size = cls.length(); CODES_CHECK(codes_set_string(handle, "class", cls.c_str(), &size), 0); + struct timeval tval_before_io, tval_after_io; eckit::Timer timer; eckit::Timer gribTimer; double elapsed_grib = 0; @@ -131,14 +146,14 @@ void FDBWrite::executeWrite(const eckit::option::CmdArgs &args) { timer.start(); - for (size_t member = 0; member < nensembles; ++member) { + for (size_t member = 1; member <= nensembles; ++member) { if (args.has("nensembles")) { - CODES_CHECK(codes_set_long(handle, "number", member+number), 0); + CODES_CHECK(codes_set_long(handle, "number", member+number-1), 0); } for (size_t step = 0; step < nsteps; ++step) { CODES_CHECK(codes_set_long(handle, "step", step), 0); - for (size_t level = 1; level <= nlevels; ++level) { - CODES_CHECK(codes_set_long(handle, "level", level), 0); + for (size_t lev = 1; lev <= nlevels; ++lev) { + CODES_CHECK(codes_set_long(handle, "level", lev+level-1), 0); for (size_t param = 1, real_param = 1; param <= nparams; ++param, ++real_param) { // GRIB API only allows us to use certain parameters while (AWKWARD_PARAMS.find(real_param) != AWKWARD_PARAMS.end()) { @@ -150,7 +165,7 @@ void FDBWrite::executeWrite(const eckit::option::CmdArgs &args) { << ", level: " << level << ", param: " << real_param << std::endl; - CODES_CHECK(codes_set_long(handle, "param", real_param), 0); + CODES_CHECK(codes_set_long(handle, "paramId", real_param), 0); CODES_CHECK(codes_get_message(handle, reinterpret_cast(&buffer), &size), 0); @@ -158,6 +173,8 @@ void FDBWrite::executeWrite(const eckit::option::CmdArgs &args) { elapsed_grib += gribTimer.elapsed(); MemoryHandle dh(buffer, size); + + if (member == 1 && step == 0 && lev == 1 && param == 1) gettimeofday(&tval_before_io, NULL); archiver.archive(dh); writeCount++; bytesWritten += size; @@ -169,6 +186,7 @@ void FDBWrite::executeWrite(const eckit::option::CmdArgs &args) { gribTimer.stop(); elapsed_grib += gribTimer.elapsed(); archiver.flush(); + if (member == nensembles && step == (nsteps - 1)) gettimeofday(&tval_after_io, NULL); gribTimer.start(); } } @@ -187,10 +205,20 @@ void FDBWrite::executeWrite(const eckit::option::CmdArgs &args) { Log::info() << "Writing duration: " << timer.elapsed() - elapsed_grib << std::endl; Log::info() << "Total rate: " << double(bytesWritten) / timer.elapsed() << " bytes / s" << std::endl; Log::info() << "Total rate: " << double(bytesWritten) / (timer.elapsed() * 1024 * 1024) << " MB / s" << std::endl; + + Log::info() << "Timestamp before first IO: " << + (long int)tval_before_io.tv_sec << "." << + std::setw(6) << std::setfill('0') << + (long int)tval_before_io.tv_usec << std::endl; + Log::info() << "Timestamp after last IO: " << + (long int)tval_after_io.tv_sec << "." << + std::setw(6) << std::setfill('0') << + (long int)tval_after_io.tv_usec << std::endl; + } -void FDBWrite::executeRead(const eckit::option::CmdArgs &args) { +void FDBHammer::executeRead(const eckit::option::CmdArgs &args) { fdb5::MessageDecoder decoder; @@ -203,25 +231,32 @@ void FDBWrite::executeRead(const eckit::option::CmdArgs &args) { size_t nensembles = args.getLong("nensembles", 1); size_t nlevels = args.getLong("nlevels"); size_t nparams = args.getLong("nparams"); + size_t number = args.getLong("number", 1); + size_t level = args.getLong("level", 1); request.setValue("expver", args.getString("expver")); request.setValue("class", args.getString("class")); + request.setValue("optimised", "on"); + + eckit::LocalConfiguration userConfig{}; + if (!args.has("disable-subtocs")) userConfig.set("useSubToc", true); + struct timeval tval_before_io, tval_after_io; eckit::Timer timer; timer.start(); fdb5::HandleGatherer handles(false); - fdb5::FDB fdb(args); + fdb5::FDB fdb(config(args, userConfig)); size_t fieldsRead = 0; for (size_t member = 1; member <= nensembles; ++member) { if (args.has("nensembles")) { - request.setValue("number", member); + request.setValue("number", member+number-1); } for (size_t step = 0; step < nsteps; ++step) { request.setValue("step", step); - for (size_t level = 1; level <= nlevels; ++level) { - request.setValue("level", level); + for (size_t lev = 1; lev <= nlevels; ++lev) { + request.setValue("levelist", lev+level-1); for (size_t param = 1, real_param = 1; param <= nparams; ++param, ++real_param) { // GRIB API only allows us to use certain parameters while (AWKWARD_PARAMS.find(real_param) != AWKWARD_PARAMS.end()) { @@ -234,6 +269,7 @@ void FDBWrite::executeRead(const eckit::option::CmdArgs &args) { << ", level: " << level << ", param: " << real_param << std::endl; + if (member == 1 && step == 0 && lev == 1 && param == 1) gettimeofday(&tval_before_io, NULL); handles.add(fdb.retrieve(request)); fieldsRead++; } @@ -245,6 +281,7 @@ void FDBWrite::executeRead(const eckit::option::CmdArgs &args) { EmptyHandle nullOutputHandle; size_t total = dh->saveInto(nullOutputHandle); + gettimeofday(&tval_after_io, NULL); timer.stop(); @@ -253,12 +290,94 @@ void FDBWrite::executeRead(const eckit::option::CmdArgs &args) { Log::info() << "Total duration: " << timer.elapsed() << std::endl; Log::info() << "Total rate: " << double(total) / timer.elapsed() << " bytes / s" << std::endl; Log::info() << "Total rate: " << double(total) / (timer.elapsed() * 1024 * 1024) << " MB / s" << std::endl; + + Log::info() << "Timestamp before first IO: " << + (long int)tval_before_io.tv_sec << "." << + std::setw(6) << std::setfill('0') << + (long int)tval_before_io.tv_usec << std::endl; + Log::info() << "Timestamp after last IO: " << + (long int)tval_after_io.tv_sec << "." << + std::setw(6) << std::setfill('0') << + (long int)tval_after_io.tv_usec << std::endl; + +} + + +void FDBHammer::executeList(const eckit::option::CmdArgs &args) { + + + std::vector minimumKeys = eckit::Resource>("FDBInspectMinimumKeys", "class,expver", true); + + fdb5::MessageDecoder decoder; + std::vector requests = decoder.messageToRequests(args(0)); + + ASSERT(requests.size() == 1); + metkit::mars::MarsRequest request = requests[0]; + + size_t nsteps = args.getLong("nsteps"); + size_t nensembles = args.getLong("nensembles", 1); + size_t nlevels = args.getLong("nlevels"); + size_t nparams = args.getLong("nparams"); + size_t number = args.getLong("number", 1); + size_t level = args.getLong("level", 1); + + request.setValue("expver", args.getString("expver")); + request.setValue("class", args.getString("class")); + + eckit::LocalConfiguration userConfig{}; + if (!args.has("disable-subtocs")) userConfig.set("useSubToc", true); + + eckit::Timer timer; + timer.start(); + + fdb5::FDB fdb(config(args, userConfig)); + fdb5::ListElement info; + + std::vector number_values; + for (size_t n = 1; n <= nensembles; ++n) { + number_values.push_back(std::to_string(n + number - 1)); + } + request.values("number", number_values); + + std::vector levelist_values; + for (size_t l = 1; l <= nlevels; ++l) { + levelist_values.push_back(std::to_string(l + level - 1)); + } + request.values("levelist", levelist_values); + + std::vector param_values; + for (size_t param = 1, real_param = 1; param <= nparams; ++param, ++real_param) { + // GRIB API only allows us to use certain parameters + while (AWKWARD_PARAMS.find(real_param) != AWKWARD_PARAMS.end()) { + real_param++; + } + param_values.push_back(std::to_string(real_param)); + } + request.values("param", param_values); + + size_t count = 0; + for (size_t step = 0; step < nsteps; ++step) { + + request.setValue("step", step); + + auto listObject = fdb.list(fdb5::FDBToolRequest(request, false, minimumKeys)); + while (listObject.next(info)) { + count++; + } + + } + + timer.stop(); + + Log::info() << "fdb-hammer - Fields listed: " << count << std::endl; + Log::info() << "fdb-hammer - List duration: " << timer.elapsed() << std::endl; + } //---------------------------------------------------------------------------------------------------------------------- int main(int argc, char **argv) { - FDBWrite app(argc, argv); + FDBHammer app(argc, argv); return app.start(); } diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 3d11f1e61..0c61129f0 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -41,11 +41,15 @@ class FDBList : public FDBVisitTool { FDBList(int argc, char **argv) : FDBVisitTool(argc, argv, "class,expver"), location_(false), + timestamp_(false), + length_(false), full_(false), porcelain_(false), json_(false) { options_.push_back(new SimpleOption("location", "Also print the location of each field")); + options_.push_back(new SimpleOption("timestamp", "Also print the timestamp when the field was indexed")); + options_.push_back(new SimpleOption("length", "Also print the field size")); options_.push_back(new SimpleOption("full", "Include all entries (including masked duplicates)")); options_.push_back(new SimpleOption("porcelain", "Streamlined and stable output for input into other tools")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); @@ -58,6 +62,8 @@ class FDBList : public FDBVisitTool { virtual void init(const CmdArgs &args); bool location_; + bool timestamp_; + bool length_; bool full_; bool porcelain_; bool json_; @@ -81,6 +87,8 @@ void FDBList::init(const CmdArgs& args) { FDBVisitTool::init(args); location_ = args.getBool("location", false); + timestamp_ = args.getBool("timestamp", false); + length_ = args.getBool("length", false); full_ = args.getBool("full", false); porcelain_ = args.getBool("porcelain", false); json_ = args.getBool("json", false); @@ -88,8 +96,8 @@ void FDBList::init(const CmdArgs& args) { if (json_) { porcelain_ = true; - if (location_) { - throw UserError("--json and --location are not compatible", Here()); + if (location_ || timestamp_ || length_) { + throw UserError("--json and --location/--timestamp/--length not compatible", Here()); } } @@ -161,7 +169,11 @@ void FDBList::execute(const CmdArgs& args) { if (json_) { (*json) << elem; } else { - elem.print(Log::info(), location_, !porcelain_); + if (porcelain_) { + elem.print(Log::info(), location_, false, false); + } else { + elem.print(Log::info(), location_, length_, timestamp_, ", "); + } Log::info() << std::endl; } } diff --git a/src/fdb5/tools/fdb-read.cc b/src/fdb5/tools/fdb-read.cc index 161ee3a29..aaceb3d07 100644 --- a/src/fdb5/tools/fdb-read.cc +++ b/src/fdb5/tools/fdb-read.cc @@ -97,7 +97,7 @@ void FDBRead::execute(const eckit::option::CmdArgs &args) { std::unique_ptr dh(handles.dataHandle()); - dh->saveInto(out); + dh->copyTo(out); } diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt index 3bb56caa6..c2353b3f8 100644 --- a/tests/fdb/CMakeLists.txt +++ b/tests/fdb/CMakeLists.txt @@ -75,3 +75,4 @@ add_subdirectory( api ) add_subdirectory( database ) add_subdirectory( tools ) add_subdirectory( type ) +add_subdirectory( daos ) \ No newline at end of file diff --git a/tests/fdb/daos/CMakeLists.txt b/tests/fdb/daos/CMakeLists.txt new file mode 100644 index 000000000..3ae4e6890 --- /dev/null +++ b/tests/fdb/daos/CMakeLists.txt @@ -0,0 +1,47 @@ +if (HAVE_DUMMY_DAOS) + + list( APPEND dummy_daos_tests + write_read + ) + + foreach( _test ${dummy_daos_tests} ) + + ecbuild_add_test( TARGET test_fdb5_dummy_daos_${_test} + SOURCES test_dummy_daos_${_test}.cc + LIBS "${DAOS_LIBRARIES}" ${DAOS_TESTS_LIBRARIES} fdb5 + INCLUDES "${DAOS_INCLUDE_DIRS}" "${DAOS_TESTS_INCLUDE_DIRS}" ) + + endforeach() + +endif() + +if (HAVE_DAOSFDB) + + list( APPEND daos_tests + daos_handle + daos_store + daos_catalogue + ) + + list( APPEND unit_test_libraries "${DAOS_LIBRARIES}" ) + list( APPEND unit_test_include_dirs "${DAOS_INCLUDE_DIRS}" ) + + if(HAVE_DAOS_ADMIN) + list( APPEND unit_test_libraries "${DAOS_TESTS_LIBRARIES}" ) + list( APPEND unit_test_include_dirs "${DAOS_TESTS_INCLUDE_DIRS}" ) + endif() + + list( APPEND unit_test_libraries fdb5 ) + + foreach( _test ${daos_tests} ) + + ecbuild_add_test( TARGET test_fdb5_daos_${_test} + SOURCES test_${_test}.cc + LIBS "${unit_test_libraries}" + INCLUDES "${unit_test_include_dirs}" + ENVIRONMENT FDB_DAOS_DMG_CONFIG_FILE=${FDB_DAOS_DMG_CONFIG_FILE} + ENVIRONMENT FDB_DAOS_TEST_POOL=${FDB_DAOS_TEST_POOL} ) + + endforeach() + +endif() diff --git a/tests/fdb/daos/test_daos_catalogue.cc b/tests/fdb/daos/test_daos_catalogue.cc new file mode 100644 index 000000000..3f03f9c4a --- /dev/null +++ b/tests/fdb/daos/test_daos_catalogue.cc @@ -0,0 +1,861 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include +#include + +#include "eckit/config/Resource.h" +#include "eckit/testing/Test.h" +#include "eckit/filesystem/URI.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/TmpDir.h" +#include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/config/YAMLConfiguration.h" + +#include "metkit/mars/MarsRequest.h" + +#include "fdb5/fdb5_config.h" +#include "fdb5/config/Config.h" +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" + +#include "fdb5/toc/TocStore.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosArrayPartHandle.h" + +#include "fdb5/daos/DaosStore.h" +#include "fdb5/daos/DaosFieldLocation.h" +#include "fdb5/daos/DaosCatalogueWriter.h" +#include "fdb5/daos/DaosCatalogueReader.h" + +using namespace eckit::testing; +using namespace eckit; + +namespace { + void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); + }; +} + +#ifdef fdb5_HAVE_DUMMY_DAOS +eckit::TmpDir& tmp_dummy_daos_root() { + static eckit::TmpDir d{}; + return d; +} +#endif + +// temporary schema,spaces,root files common to all DAOS Catalogue tests + +eckit::TmpFile& schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::TmpFile& opt_schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::PathName& catalogue_tests_tmp_root() { + static eckit::PathName cd("./daos_catalogue_tests_fdb_root"); + return cd; +} + +namespace fdb { +namespace test { + +CASE( "Setup" ) { + +#ifdef fdb5_HAVE_DUMMY_DAOS + tmp_dummy_daos_root().mkdir(); + ::setenv("DUMMY_DAOS_DATA_ROOT", tmp_dummy_daos_root().path().c_str(), 1); +#endif + + // ensure fdb root directory exists. If not, then that root is + // registered as non existing and Catalogue/Store tests fail. + if (catalogue_tests_tmp_root().exists()) deldir(catalogue_tests_tmp_root()); + catalogue_tests_tmp_root().mkdir(); + ::setenv("FDB_ROOT_DIRECTORY", catalogue_tests_tmp_root().path().c_str(), 1); + + // prepare schema for tests involving DaosCatalogue + + std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; + + std::unique_ptr hs(schema_file().fileHandle()); + hs->openForWrite(schema_str.size()); + { + eckit::AutoClose closer(*hs); + hs->write(schema_str.data(), schema_str.size()); + } + + std::string opt_schema_str{"[ a, b [ c?, d [ e?, f ]]]"}; + + std::unique_ptr hs_opt(opt_schema_file().fileHandle()); + hs_opt->openForWrite(opt_schema_str.size()); + { + eckit::AutoClose closer(*hs_opt); + hs_opt->write(opt_schema_str.data(), opt_schema_str.size()); + } + + // this is necessary to avoid ~fdb/etc/fdb/schema being used where + // LibFdb5::instance().defaultConfig().schema() is called + // due to no specified schema file (e.g. in Key::registry()) + ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); + +} + +CASE("DaosCatalogue tests") { + + // test parameters + + std::string root_cont_name{"test_root"}; + int container_oids_per_alloc = 1000; +#if defined(fdb5_HAVE_DAOS_ADMIN) || defined(fdb5_HAVE_DUMMY_DAOS) + std::string pool_name{"fdb_pool2"}; +#else + std::string pool_name; + pool_name = eckit::Resource( + "fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name + ); + EXPECT(pool_name.length() > 0); +#endif + + // bootstrap daos + + fdb5::UUID pool_uuid; + { + /// @btodo: should DaosManager really be configured here? May be better not to configure it + /// here, only specify client config in FDB config file, and let the constructor of + /// DaosStore or DaosCatalogue configure DaosManager. + fdb5::DaosManager::instance().configure( + eckit::LocalConfiguration(YAMLConfiguration( + "container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + )) + ); + fdb5::DaosSession s{}; + +#ifdef fdb5_HAVE_DAOS_ADMIN + fdb5::DaosPool& pool = s.createPool(pool_name); +#else + #ifdef fdb5_HAVE_DUMMY_DAOS + std::string pool_uuid_str{"00000000-0000-0000-0000-000000000004"}; + (tmp_dummy_daos_root() / pool_uuid_str).mkdir(); + ::symlink( + (tmp_dummy_daos_root() / pool_uuid_str).path().c_str(), + (tmp_dummy_daos_root() / pool_name).path().c_str() + ); + #endif + fdb5::DaosPool& pool = s.getPool(pool_name); +#endif + + pool_uuid = pool.uuid(); + } + + SECTION("DaosCatalogue archive (index) and retrieve without a Store") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "daos:\n" + " catalogue:\n" + " pool: " + pool_name + "\n" + " root_cont: " + root_cont_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + fdb5::Schema schema{schema_file()}; + + /// @note: a=11,b=22 instead of a=1,b=2 to avoid collision with potential parallel runs of store tests using a=1,b=2 + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}, schema.registry()); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}, schema.registry()); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}, schema.registry()); + + // archive + + /// DaosManager is configured with client config from the file + std::unique_ptr loc(new fdb5::DaosFieldLocation( + eckit::URI{"daos", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key(nullptr, true) + )); + + { + fdb5::DaosCatalogueWriter dcatw{db_key, config}; + + fdb5::DaosName db_cont{pool_name, db_key.valuesToString()}; + fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + EXPECT(db_cont.exists()); + EXPECT(cat_kv.exists()); + + fdb5::Catalogue& cat = dcatw; + cat.selectIndex(index_key); + fdb5::DaosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from config + fdb5::DaosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; + EXPECT(index_kv.exists()); + EXPECT(cat_kv.has(index_key.valuesToString())); + + fdb5::CatalogueWriter& catw = dcatw; + catw.archive(field_key, std::move(loc)); + EXPECT(index_kv.has(field_key.valuesToString())); + fdb5::DaosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; + fdb5::DaosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; + EXPECT(e_axis_kv.exists()); + EXPECT(e_axis_kv.has("5")); + fdb5::DaosKeyValueOID f_axis_kv_oid{index_key.valuesToString() + std::string{".f"}, OC_S1}; + fdb5::DaosKeyValueName f_axis_kv{pool_name, db_key.valuesToString(), f_axis_kv_oid}; + EXPECT(f_axis_kv.exists()); + EXPECT(f_axis_kv.has("6")); + } + + // retrieve + + { + fdb5::DaosCatalogueReader dcatr{db_key, config}; + + fdb5::Catalogue& cat = dcatr; + cat.selectIndex(index_key); + + fdb5::Field f; + fdb5::CatalogueReader& catr = dcatr; + catr.retrieve(field_key, f); + EXPECT(f.location().uri().name() == eckit::URI("daos", "test_uri").name()); + EXPECT(f.location().offset() == eckit::Offset(0)); + EXPECT(f.location().length() == eckit::Length(1)); + } + + // remove (manual deindex) + + { + fdb5::DaosCatalogueWriter dcatw{db_key, config}; + fdb5::DaosName db_cont{dcatw.uri()}; + std::ostream out(std::cout.rdbuf()); + + fdb5::DaosCatalogue::remove(db_cont, out, out, true); + + fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + EXPECT_NOT(cat_kv.exists()); + EXPECT_NOT(db_cont.exists()); + } + + } + + SECTION("DaosCatalogue archive (index) and retrieve with a DaosStore") { + + // FDB configuration + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "daos:\n" + " store:\n" + " pool: " + pool_name + "\n" + " catalogue:\n" + " pool: " + pool_name + "\n" + " root_cont: " + root_cont_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // store data + + char data[] = "test"; + + fdb5::DaosStore dstore{schema, db_key, config}; + fdb5::Store& store = static_cast(dstore); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + /// @todo: there are two cont create with label here + /// @todo: again, daos_fini happening before cont and pool close + + // index data + + { + fdb5::DaosCatalogueWriter dcatw{db_key, config}; + fdb5::Catalogue& cat = dcatw; + cat.deselectIndex(); + cat.selectIndex(index_key); + fdb5::CatalogueWriter& catw = dcatw; + catw.archive(field_key, std::move(loc)); + + /// flush store before flushing catalogue + dstore.flush(); // not necessary if using a DAOS store + } + + // find data + + fdb5::Field field; + { + fdb5::DaosCatalogueReader dcatr{db_key, config}; + fdb5::Catalogue& cat = dcatr; + cat.selectIndex(index_key); + fdb5::CatalogueReader& catr = dcatr; + catr.retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // deindex data + + { + fdb5::DaosCatalogueWriter dcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(dcat); + std::ostream out(std::cout.rdbuf()); + metkit::mars::MarsRequest r = db_key.request("retrieve"); + std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + cat.visitEntries(*wv, store, false); + } + + /// @todo: again, daos_fini happening before + + } + + SECTION("DaosCatalogue archive (index) and retrieve with a TocStore") { + + // FDB configuration + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "daos:\n" + " catalogue:\n" + " pool: " + pool_name + "\n" + " root_cont: " + root_cont_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // store data + + char data[] = "test"; + + fdb5::TocStore tstore{schema, db_key, config}; + fdb5::Store& store = static_cast(tstore); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + /// @todo: there are two cont create with label here + /// @todo: again, daos_fini happening before cont and pool close + + // index data + + { + fdb5::DaosCatalogueWriter dcatw{db_key, config}; + fdb5::Catalogue& cat = dcatw; + cat.deselectIndex(); + cat.selectIndex(index_key); + fdb5::CatalogueWriter& catw = dcatw; + catw.archive(field_key, std::move(loc)); + + /// flush store before flushing catalogue + tstore.flush(); + } + + // find data + + fdb5::Field field; + { + fdb5::DaosCatalogueReader dcatr{db_key, config}; + fdb5::Catalogue& cat = dcatr; + cat.selectIndex(index_key); + fdb5::CatalogueReader& catr = dcatr; + catr.retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + + std::vector test(dh->size()); + dh->openForRead(); + { + eckit::AutoClose closer(*dh); + dh->read(&test[0], test.size() - 3); + } + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // remove data + + /// @todo: should DaosStore::remove accept full URIs to field arrays and remove the store container? + eckit::PathName store_path{field.location().uri().path()}; + std::ostream out(std::cout.rdbuf()); + store.remove(field.location().uri(), out, out, false); + EXPECT(store_path.exists()); + store.remove(field.location().uri(), out, out, true); + EXPECT_NOT(store_path.exists()); + + // deindex data + + { + fdb5::DaosCatalogueWriter dcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(dcat); + std::ostream out(std::cout.rdbuf()); + metkit::mars::MarsRequest r = db_key.request("retrieve"); + std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + cat.visitEntries(*wv, store, false); + } + + /// @todo: again, daos_fini happening before + + } + + SECTION("Via FDB API with a DAOS catalogue and store") { + + // FDB configuration + + int container_oids_per_alloc_small = 100; + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: daos\n" + "store: daos\n" + "daos:\n" + " catalogue:\n" + " pool: " + pool_name + "\n" + " root_cont: " + root_cont_name + "\n" + " store:\n" + " pool: " + pool_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc_small) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest all_req{ + metkit::mars::MarsRequest{}, + true, + std::vector{} + }; + + // initialise FDB + + fdb5::FDB fdb(config); + + // check FDB is empty + + size_t count; + fdb5::ListElement info; + + /// @todo: here, DaosManager is being configured with DAOS client config passed to FDB instance constructor. + // It happens in EntryVisitMechanism::visit when calling DB::open. Is this OK, or should this configuring + // rather happen as part of transforming a FieldLocation into a DataHandle? It is probably OK. One thing + // is to configure the DAOS client and the other thing is to initialise it. + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // archive data + + char data[] = "test"; + + /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times below. + // Should this be avoided? + fdb.archive(request_key, data, sizeof(data)); + + fdb.flush(); + + // retrieve data + + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // list all + + listObject = fdb.list(all_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // wipe data + + fdb5::WipeElement elem; + + // dry run attempt to wipe with too specific request + + auto wipeObject = fdb.wipe(full_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count == 0); + + // dry run wipe index and store unit + wipeObject = fdb.wipe(index_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + + // dry run wipe database + wipeObject = fdb.wipe(db_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + + // ensure field still exists + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // attempt to wipe with too specific request + wipeObject = fdb.wipe(full_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count == 0); + /// @todo: really needed? + fdb.flush(); + + // wipe index and store unit + wipeObject = fdb.wipe(index_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) count++; + EXPECT(count == 0); + + /// @todo: ensure index and corresponding container do not exist + /// @todo: ensure DB still exists + /// @todo: list db or index and expect count = 0? + + // re-archive data + + /// @note: FDB holds a LocalFDB which holds an Archiver which holds open DBs (DaosCatalogueWriters). + /// If a whole DB is wiped, the top-level structures for that DB (main and catalogue KVs in this case) + /// are deleted. If willing to archive again into that DB, the DB needs to be constructed again as the + /// top-level structures are only generated as part of the DaosCatalogueWriter constructor. There is + /// no way currently to destroy the open DBs held by FDB other than entirely destroying FDB. + /// Alternatively, a separate FDB instance can be created. + fdb5::FDB fdb2(config); + + fdb2.archive(request_key, data, sizeof(data)); + + fdb2.flush(); + + listObject = fdb2.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // wipe full database + + wipeObject = fdb2.wipe(db_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + /// @todo: really needed? + fdb2.flush(); + + // ensure field does not exist + + listObject = fdb2.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 0); + + /// @todo: ensure DB and corresponding pool do not exist + + /// @todo: ensure new DaosSession has updated daos client config + + } + + SECTION("OPTIONAL SCHEMA KEYS") { + + // FDB configuration + + ::setenv("FDB_SCHEMA_FILE", opt_schema_file().path().c_str(), 1); + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "type: local\n" + "schema : " + opt_schema_file().path() + "\n" + "engine: daos\n" + "store: daos\n" + "daos:\n" + " catalogue:\n" + " pool: " + pool_name + "\n" + " root_cont: " + root_cont_name + "\n" + " store:\n" + " pool: " + pool_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"d", "4"}, {"f", "6"}}); + fdb5::Key request_key2({{"a", "11"}, {"b", "22"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest full_req2{ + request_key2.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest all_req{ + metkit::mars::MarsRequest{}, + true, + std::vector{} + }; + + // initialise FDB + + fdb5::FDB fdb(config); + + // check FDB is empty + + size_t count; + fdb5::ListElement info; + + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // archive data with incomplete key + + char data[] = "test"; + + fdb.archive(request_key, data, sizeof(data)); + + fdb.flush(); + + // list data + + listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 1); + + // retrieve data + + { + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + } + + // archive data with complete key + + char data2[] = "abcd"; + + fdb.archive(request_key2, data2, sizeof(data)); + + fdb.flush(); + + // list data + + listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 2); + + // retrieve data + + { + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + } + + { + metkit::mars::MarsRequest r = request_key2.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data2))); + EXPECT(::memcmp(mh.data(), data2, sizeof(data2)) == 0); + } + + } + + // teardown daos + +#ifdef fdb5_HAVE_DAOS_ADMIN + /// AutoPoolDestroy is not possible here because the pool is + /// created above with an ephemeral session + fdb5::DaosSession().destroyPool(pool_uuid); +#else + for (auto& c : fdb5::DaosSession().getPool(pool_uuid).listContainers()) + if (c == root_cont_name || c == "11:22") + fdb5::DaosSession().getPool(pool_uuid).destroyContainer(c); +#endif + +} + +} // namespace test +} // namespace fdb + +int main(int argc, char **argv) +{ + return run_tests ( argc, argv ); +} diff --git a/tests/fdb/daos/test_daos_handle.cc b/tests/fdb/daos/test_daos_handle.cc new file mode 100644 index 000000000..15c5af84b --- /dev/null +++ b/tests/fdb/daos/test_daos_handle.cc @@ -0,0 +1,744 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include +#include + +#include "eckit/config/Resource.h" +#include "eckit/testing/Test.h" +#include "eckit/filesystem/URI.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/TmpDir.h" +#include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/config/YAMLConfiguration.h" + +#include "fdb5/fdb5_config.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosContainer.h" +#include "fdb5/daos/DaosObject.h" +#include "fdb5/daos/DaosName.h" +#include "fdb5/daos/DaosArrayHandle.h" +#include "fdb5/daos/DaosException.h" + +using namespace eckit::testing; +using namespace eckit; + +#ifdef fdb5_HAVE_DUMMY_DAOS +eckit::TmpDir& tmp_dummy_daos_root() { + static eckit::TmpDir d{}; + return d; +} +#endif + +bool endsWith(const std::string& str, const std::string& end) { + return 0 == str.compare(str.length() - end.length(), end.length(), end); +} + +namespace fdb { +namespace test { + +#ifdef fdb5_HAVE_DUMMY_DAOS +CASE( "Setup" ) { + + tmp_dummy_daos_root().mkdir(); + ::setenv("DUMMY_DAOS_DATA_ROOT", tmp_dummy_daos_root().path().c_str(), 1); + +} +#endif + +#ifdef fdb5_HAVE_DAOS_ADMIN +CASE( "DaosPool" ) { + + /// @todo: currently, all pool and container connections are cached and kept open for the duration of the process. Would + /// be nice to close container connections as they become unused. However the DaosContainer instances are managed by the + /// DaosPools/DaosSession, so we never know when the user has finished using a certain container. My current thought is + /// we don't need to fix this, as each process will only use a single pool and 2 * (indices involved) containers. + /// However in a large parallel application, while all client processes are running, there may be a large number of + /// open containers in the DAOS system. One idea would be to use shared pointers to count number of uses. + + /// @todo: A declarative approach would be better in my opinion. + /// The current approach is an imperative one, where DaosObject and DaosContainer instances always represent existing + /// entities in DAOS from the instant they are created. In highly parallel workflows, validity of such instances will be + /// ephemeral, and by the time we perform an action on them, the DAOS entity they represent may no longer exist. In the + /// declarative approach, the containers and objects would be opened right before the action and fail if they don't exist. + /// In the imperative approach they would fail as well, but the initial checks performed to ensure existence of the DAOS + /// entities would be useless and degrade performance. + + /// @todo: cpp uuid wrapper, to avoid weird headers + + /// @todo: do not return iterators in e.g. DaosSession::getCachedPool. Return DaosPool& + + /// @todo: replace deque by map + + /// @note: using hard-coded config defaults in DaosManager + fdb5::DaosSession s{}; + + SECTION("unnamed pool") { + + fdb5::DaosPool& pool = s.createPool(); /// admin function, not usually called in the client code + fdb5::AutoPoolDestroy destroyer(pool); + + std::cout << pool.name() << std::endl; + + EXPECT(pool.name().size() == 36); + + /// @todo: there's an attempt to close unopened pool here due to the pool destruction mechanism + + } + + SECTION("named pool") { + + std::string pool_name{"test_pool_1"}; + + fdb5::DaosPool& pool = s.createPool(pool_name); + fdb5::AutoPoolDestroy destroyer(pool); + + std::cout << pool.name() << std::endl; + + EXPECT(pool.name() == pool_name); + + fdb5::DaosPool& pool_h = s.getPool(pool_name); + + EXPECT(&pool == &pool_h); + + } + + SECTION("pool uuid actions") { + + fdb5::DaosPool& pool = s.createPool(); + fdb5::AutoPoolDestroy destroyer(pool); + + const fdb5::UUID& pool_uuid = pool.uuid(); + + char uuid_cstr[37]; + uuid_unparse(pool_uuid.internal, uuid_cstr); + std::cout << uuid_cstr << std::endl; + + } + + /// @todo: test passing some session ad-hoc config for DAOS client + + /// @todo: there's an extra pair of daos_init and daos_fini happening here + +} +#endif + +CASE( "DaosContainer, DaosArray and DaosKeyValue" ) { + + std::string cont_name{"test_cont_2"}; + + /// again using hard-coded config defaults in DaosManager + fdb5::DaosSession s{}; + +#ifdef fdb5_HAVE_DAOS_ADMIN + std::string pool_name{"test_pool_2"}; + fdb5::DaosPool& pool = s.createPool(pool_name); + fdb5::AutoPoolDestroy destroyer(pool); + fdb5::DaosContainer& cont = pool.createContainer(cont_name); +#else + #ifdef fdb5_HAVE_DUMMY_DAOS + std::string pool_uuid{"00000000-0000-0000-0000-000000000001"}; + std::string pool_name{"test_pool_2"}; + (tmp_dummy_daos_root() / pool_uuid).mkdir(); + ::symlink( + (tmp_dummy_daos_root() / pool_uuid).path().c_str(), + (tmp_dummy_daos_root() / pool_name).path().c_str() + ); + #else + std::string pool_name; + pool_name = eckit::Resource( + "fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name + ); + EXPECT(pool_name.length() > 0); + #endif + fdb5::DaosPool& pool = s.getPool(pool_name); + fdb5::DaosContainer& cont = pool.createContainer(cont_name); + fdb5::AutoContainerDestroy destroyer(cont); +#endif + + SECTION("named container") { + + std::cout << cont.name() << std::endl; + + EXPECT(cont.name() == cont_name); + + fdb5::DaosContainer& cont_h = pool.getContainer(cont_name); + + EXPECT(&cont == &cont_h); + + EXPECT_THROWS_AS(pool.getContainer("nonexisting_cont"), fdb5::DaosEntityNotFoundException); + + std::vector cont_list(pool.listContainers()); + /// @note: not compatible with parallel test runs on a same pool (if !have_FDB_DAOS_ADMIN) + /// EXPECT(cont_list.size() == 1); + /// EXPECT(cont_list.front() == cont_name); + + /// @todo: two attempts to close unopened containers here. This is due to mechanism triggered upon + /// pool destroy to ensure all matching container handles in the session cache are closed. + /// One way to address this would be to have a flag expectOpen = true in DaosContainer::close(). + /// Whenever close() is called as part of pool destroy or batch container close, a false value + /// could be passed to the close() method to avoid logging a warning message. + + } + + SECTION("unnamed object") { + + // create new object with new automatically allocated oid + fdb5::DaosArray arr = cont.createArray(); + std::cout << "New automatically allocated Array OID: " << arr.name() << std::endl; + fdb5::DaosKeyValue kv = cont.createKeyValue(); + std::cout << "New automatically allocated KeyValue OID: " << kv.name() << std::endl; + + /// @todo: + // arr.destroy(); + // kv.destroy(); + + } + + SECTION("DaosOID and named object") { + + // create new object with oid generated from user input + uint32_t hi = 0x00000001; + uint64_t lo = 0x0000000000000001; + fdb5::DaosOID oid{hi, lo, DAOS_OT_ARRAY, OC_S1}; + fdb5::DaosArray write_obj = cont.createArray(oid); + + std::string id_string = write_obj.name(); + std::cout << "New user-spec-based OID: " << id_string << std::endl; + EXPECT(id_string.length() == 32); + /// @todo: do these checks numerically. Also test invalid characters, etc. + std::string end{"000000010000000000000001"}; + EXPECT(0 == id_string.compare(id_string.length() - end.length(), end.length(), end)); + + // represent existing object with known oid + fdb5::DaosOID read_id{id_string}; + fdb5::DaosArray read_obj{cont, read_id}; + + // attempt access non-existing object + fdb5::DaosArrayOID oid_ne{0, 0, OC_S1}; + oid_ne.generateReservedBits(cont); + EXPECT_THROWS_AS(fdb5::DaosArray obj(cont, oid_ne), fdb5::DaosEntityNotFoundException); + + /// @todo: + //write_obj.destroy(); + + } + + /// @todo: DaosArray write and read + + SECTION("DaosKeyValue put and get") { + + fdb5::DaosKeyValue kv = cont.createKeyValue(); + + std::string test_key{"test_key_1"}; + + char data[] = "test_data_1"; + kv.put(test_key, data, (uint64_t) sizeof(data)); + + uint64_t size = kv.size(test_key); + EXPECT(size == (uint64_t) sizeof(data)); + + uint64_t res; + char read_data[20] = ""; + res = kv.get(test_key, read_data, sizeof(read_data)); + EXPECT(res == size); + EXPECT(std::memcmp(data, read_data, sizeof(data)) == 0); + + EXPECT(!kv.has("nonexisting_key")); + EXPECT(kv.size("nonexisting_key") == 0); + EXPECT_THROWS_AS(kv.get("nonexisting_key", nullptr, 0), fdb5::DaosEntityNotFoundException); + + /// @todo: + //kv.destroy(); + + } + + SECTION("DaosName, DaosArrayName, DaosKeyValueName") { + + /// @todo: is the private ctor of DaosArray and DaosKeyValue kept private? + + fdb5::DaosName np{pool_name}; + EXPECT(np.exists()); + EXPECT(np.URI().asString() == std::string("daos:") + pool_name); + + fdb5::DaosName np_ne{"a"}; + EXPECT_NOT(np_ne.exists()); + + fdb5::DaosName nc{pool_name, cont_name}; + EXPECT(nc.exists()); + EXPECT(nc.URI().asString() == std::string("daos:") + pool_name + "/" + cont_name); + + fdb5::DaosName nc_ne{"a", "b"}; + EXPECT_NOT(nc_ne.exists()); + + fdb5::DaosOID test_oid{1, 2, DAOS_OT_ARRAY, OC_S1}; + fdb5::DaosName n1{pool_name, cont_name, test_oid}; + EXPECT(n1.poolName() == pool_name); + EXPECT(n1.hasContainerName()); + EXPECT(n1.containerName() == cont_name); + EXPECT(n1.hasOID()); + EXPECT_NOT(test_oid.wasGenerated()); + fdb5::DaosOID test_oid_gen = n1.OID(); + EXPECT(test_oid_gen.wasGenerated()); + EXPECT_NOT(n1.exists()); + n1.create(); + EXPECT(n1.exists()); + std::string start{std::string("daos:") + pool_name + "/" + cont_name + "/"}; + std::string end{"000000010000000000000002"}; + std::string n1_uri_str = n1.URI().asString(); + EXPECT(0 == n1_uri_str.compare(0, start.length(), start)); + EXPECT(0 == n1_uri_str.compare(n1_uri_str.length() - end.length(), end.length(), end)); + EXPECT(n1.size() == eckit::Length(0)); + + fdb5::DaosName nc_new{pool_name, "new_cont"}; + EXPECT_NOT(nc_new.exists()); + fdb5::DaosArrayName na_new = nc_new.createArrayName(); + EXPECT(nc_new.exists()); + EXPECT_NOT(na_new.exists()); + na_new.create(); + EXPECT(na_new.exists()); + /// @todo: + // na_new.destroy(); + // EXPECT_NOT(na_new.exists()); + // EXPECT(nc_new.exists()); + nc_new.destroy(); + EXPECT_NOT(nc_new.exists()); + + std::string test_oid_2_str{"00000000000000020000000000000002"}; + fdb5::DaosOID test_oid_2{test_oid_2_str}; + fdb5::DaosName n2("a", "b", test_oid_2); + EXPECT(n2.asString() == "a/b/" + test_oid_2_str); + + eckit::URI u2("daos", "a/b/" + test_oid_2_str); + fdb5::DaosName n3(u2); + EXPECT(n3.asString() == "a/b/" + test_oid_2_str); + EXPECT(n3.URI() == u2); + + fdb5::DaosArray arr{cont, n1.OID()}; + fdb5::DaosName n4{arr}; + EXPECT(n4.exists()); + + eckit::URI uri = n1.URI(); + EXPECT(arr.URI() == uri); + + /// @todo: + // n1.destroy(); + // EXPECT_NOT(n1.exists()); + // EXPECT_NOT(n4.exists()); + + fdb5::DaosKeyValueName nkv = nc.createKeyValueName(); + /// @todo: currently, existence of kvs is checked by attempting open. + /// But a DAOS kv open creates it if not exists. Should implement a better kv + /// existency check not involving open/create before uncommenting the following test. + // EXPECT_NOT(nkv.exists()); + nkv.create(); + EXPECT(nkv.exists()); + fdb5::DaosKeyValue kv{s, nkv}; + char data[] = "test_value_3"; + kv.put("test_key_3", data, (uint64_t) sizeof(data)); + EXPECT(nkv.has("test_key_3")); + /// @todo: + // nkv.destroy(); + // EXPECT_NOT(nkv.exists()); + + /// @todo: serialise + + /// @todo: deserialise + fdb5::DaosName deserialisedname(pool_name, cont_name, n1.OID()); + + std::cout << "Object size is: " << deserialisedname.size() << std::endl; + + kv.put("test_key_4", data, sizeof(data)); + EXPECT(nkv.has("test_key_4")); + std::vector keys = kv.keys(); + EXPECT(keys.size() == 2); + EXPECT(keys[0] == "test_key_3"); + EXPECT(keys[1] == "test_key_4"); + + } + + SECTION("DaosHandle write, append and read") { + + fdb5::DaosOID test_oid{1, 3, DAOS_OT_ARRAY, OC_S1}; + fdb5::DaosArrayName test_name{pool_name, cont_name, test_oid}; + + char data[] = "test_data_2"; + long res; + + fdb5::DaosArrayHandle h{test_name}; + /// @todo: this triggers array create but does not wipe existing array if any + h.openForWrite(Length(sizeof(data))); + { + eckit::AutoClose closer(h); + res = h.write(data, (uint64_t) sizeof(data)); + EXPECT(res == (uint64_t) sizeof(data)); + EXPECT(h.position() == Offset(sizeof(data))); + + res = h.write(data, (uint64_t) sizeof(data)); + EXPECT(res == (uint64_t) sizeof(data)); + EXPECT(h.position() == Offset(2 * sizeof(data))); + } + + h.flush(); + + eckit::URI u{std::string("daos:") + pool_name + "/" + cont_name + "/" + test_name.OID().asString()}; + fdb5::DaosArrayName read_name{u}; + + fdb5::DaosArrayHandle h2(read_name); + Length t = h2.openForRead(); + EXPECT(t == Length(2 * sizeof(data))); + EXPECT(h2.position() == Offset(0)); + std::vector read_data((size_t) t, 0); + { + eckit::AutoClose closer(h2); + for (int i = 0; i < 2; ++i) { + res = h2.read(&read_data[0] + i * sizeof(data), (uint64_t) sizeof(data)); + EXPECT(res == (uint64_t) sizeof(data)); + } + EXPECT(h2.position() == Offset(2 * sizeof(data))); + } + EXPECT(std::memcmp(data, &read_data[0], sizeof(data)) == 0); + EXPECT(std::memcmp(data, &read_data[0] + sizeof(data), sizeof(data)) == 0); + + std::unique_ptr h3(read_name.dataHandle()); + Length t2 = h3->openForRead(); + std::vector read_data2((size_t) t2, 0); + { + eckit::AutoClose closer(*h3); + for (int i = 0; i < 2; ++i) { + h3->read(&read_data2[0] + i * sizeof(data), (uint64_t) sizeof(data)); + } + } + EXPECT(std::memcmp(data, &read_data2[0], sizeof(data)) == 0); + EXPECT(std::memcmp(data, &read_data2[0] + sizeof(data), sizeof(data)) == 0); + + fdb5::DaosArrayHandle dh_fail( + fdb5::DaosArrayName( + pool_name, cont_name, fdb5::DaosOID{1, 0, DAOS_OT_ARRAY, OC_S1} + ) + ); + EXPECT_THROWS_AS(dh_fail.openForRead(), fdb5::DaosEntityNotFoundException); + + /// @todo: pool, container and object opening are optional before calling + /// DaosHandle::openForRead. Test it. + /// @todo: container and object creation are optional before calling + /// DaosHandle::openForWrite. Test it. + /// @todo: test unopen/uncreated DaosObject::size() + + } + + /// @note: There's no actual need for container destruction as either pool or + /// container are autodestroyed depending on if_HAVE_DAOS_ADMIN. + + /// @todo: when enabling this, AutoPoolDestroy fails as a corresponding DaosContainer instance + /// still exists in the DaosPool, but the corresponding container does not exist and + /// dummy_daos destroy fails trying to rename an unexisting directory. The issue of pool + /// and container destruction and invalidation needs to be addressed first. + // cont.destroy(); + + /// @todo: test open pool destroy ensure it is closed + +} + +/// @todo: test a new case where some DAOS operations are carried out with a DaosSession with specific config +/// overriding (but not rewriting) DaosManager defaults + +CASE( "DaosName and DaosHandle workflows" ) { + + std::string cont_name{"test_cont_3"}; + + /// @note: again using hard-coded config defaults in DaosManager + fdb5::DaosSession s{}; + +#ifdef fdb5_HAVE_DAOS_ADMIN + std::string pool_name{"test_pool_3"}; + fdb5::DaosPool& pool = s.createPool(pool_name); + fdb5::AutoPoolDestroy destroyer(pool); + fdb5::DaosContainer& cont = pool.createContainer(cont_name); +#else + #ifdef fdb5_HAVE_DUMMY_DAOS + std::string pool_uuid{"00000000-0000-0000-0000-000000000002"}; + std::string pool_name{"test_pool_3"}; + (tmp_dummy_daos_root() / pool_uuid).mkdir(); + ::symlink( + (tmp_dummy_daos_root() / pool_uuid).path().c_str(), + (tmp_dummy_daos_root() / pool_name).path().c_str() + ); + #else + std::string pool_name; + pool_name = eckit::Resource( + "fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name + ); + EXPECT(pool_name.length() > 0); + #endif + fdb5::DaosPool& pool = s.getPool(pool_name); + fdb5::DaosContainer& cont = pool.createContainer(cont_name); + fdb5::AutoContainerDestroy destroyer(cont); +#endif + + char data[] = "test_data_3"; + eckit::Length len{sizeof(data)}; + + long res; + + SECTION("Array write to existing pool and container, with ungenerated OID") { + + fdb5::DaosArrayOID oid{3, 1, OC_S1}; + fdb5::DaosArrayName na{pool_name, cont_name, oid}; + std::unique_ptr h(na.dataHandle()); + /// @todo: cast to reference, and dereference ptr with * + EXPECT(dynamic_cast(h.get())); + h->openForWrite(len); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + EXPECT_THROWS_AS(oid.asString(), eckit::AssertionFailed); /// ungenerated + std::string oid_end{"000000030000000000000001"}; + EXPECT(endsWith(na.OID().asString(), oid_end)); + EXPECT(na.size() == len); + + } + + SECTION("Array write to existing pool and container, with automatically generated OID") { + + fdb5::DaosName n{pool_name, cont_name}; + fdb5::DaosArrayName na = n.createArrayName(); + std::unique_ptr h(na.dataHandle()); + h->openForWrite(len); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + std::cout << "Generated OID: " << na.OID().asString() << std::endl; + EXPECT(na.size() == len); + + } + + SECTION("Array write to existing pool and container, with URI") { + + eckit::URI container{std::string("daos:") + pool_name + "/" + cont_name}; + fdb5::DaosName n{container}; + fdb5::DaosArrayName na = n.createArrayName(); + std::unique_ptr h(na.dataHandle()); + h->openForWrite(len); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + std::cout << "Generated OID: " << na.OID().asString() << std::endl; + EXPECT(na.size() == len); + + } + + SECTION("Array write/read to/from existing pool and container, with generated OID") { + + /// write + fdb5::DaosArrayOID oid{3, 2, OC_S1}; + oid.generateReservedBits(cont); + fdb5::DaosArrayName na{pool_name, cont_name, oid}; + // fdb5::DaosArrayName n{pool_name, cont_name, {"00110000000000000000000000000001"}}; + std::unique_ptr h(na.dataHandle()); + h->openForWrite(len); /// @todo: creates it if not exists, should it fail? + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + std::string oid_end{"000000030000000000000002"}; + EXPECT(endsWith(na.OID().asString(), oid_end)); + EXPECT(na.size() == len); + + } + + SECTION("Array write to existing pool but non-existing container, with ungenerated OID") { + + fdb5::DaosArrayOID oid{3, 3, OC_S1}; + fdb5::DaosArrayName na{pool_name, "new_cont_1", oid}; + EXPECT_NOT(fdb5::DaosName(pool_name, "new_cont_1").exists()); + std::unique_ptr h(na.dataHandle()); /// @todo: should dataHandle() assert pool exists? + h->openForWrite(len); + fdb5::AutoContainerDestroy d(pool.getContainer("new_cont_1")); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + EXPECT_THROWS_AS(oid.asString(), eckit::AssertionFailed); /// ungenerated + std::string oid_end{"000000030000000000000003"}; + EXPECT(endsWith(na.OID().asString(), oid_end)); + EXPECT(na.size() == len); + EXPECT(fdb5::DaosName(pool_name, "new_cont_1").exists()); + + } + + SECTION("Array write to existing pool but non-existing container, with automatically generated OID") { + + fdb5::DaosName n{pool_name, "new_cont_2"}; + fdb5::DaosArrayName na = n.createArrayName(); + EXPECT(n.exists()); + EXPECT_NOT(na.exists()); + std::unique_ptr h(na.dataHandle()); + h->openForWrite(len); + fdb5::AutoContainerDestroy d(pool.getContainer("new_cont_2")); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + std::cout << "Generated OID: " << na.OID().asString() << std::endl; + EXPECT(na.size() == len); + EXPECT(fdb5::DaosName(pool_name, "new_cont_2").exists()); + + } + + /// @todo: should use of a non-existing container plus generated OID be forbidden? + /// In principle it should be forbidden, as an OID can only be generated via existing enclosing container. + /// However it would not be straightforward to implement with the curren design of DaosOID and DaosName. + /// For now it is allowed. The container and object will be created in openForWrite() if not exist. + SECTION("Array write to existing pool but non-existing container, with generated OID") { + + fdb5::DaosName nc{pool_name, "new_cont_3"}; + fdb5::DaosArrayName na = nc.createArrayName(OC_S1); + nc.destroy(); + EXPECT_NOT(nc.exists()); + std::unique_ptr h(na.dataHandle()); + h->openForWrite(len); + fdb5::AutoContainerDestroy d(pool.getContainer("new_cont_3")); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + EXPECT(nc.exists()); + EXPECT(na.size() == len); + + } + + SECTION("Array read from existing pool and container, with generated OID") { + + fdb5::DaosArrayOID oid{3, 4, OC_S1}; + fdb5::DaosArrayName na{pool_name, cont_name, oid}; + std::unique_ptr h(na.dataHandle()); + h->openForWrite(len); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + EXPECT(na.size() == len); + + /// existing generated OID + fdb5::DaosArrayName na_read{pool_name, cont_name, na.OID()}; + std::unique_ptr h2(na_read.dataHandle()); + Length t = h2->openForRead(); + std::vector read_data((size_t) t, 0); + { + eckit::AutoClose closer(*h2); + EXPECT(eckit::Length(read_data.size()) >= h2->size()); + res = h2->read(&read_data[0], h2->size()); + } + EXPECT(res == len); + EXPECT(std::memcmp(data, &read_data[0], sizeof(data)) == 0); + + /// non-existing generated OID + fdb5::DaosArrayOID oid_ne{3, 5, OC_S1}; + oid_ne.generateReservedBits(cont); + fdb5::DaosArrayName na_read_ne{pool_name, cont_name, oid_ne}; + std::unique_ptr h3(na_read_ne.dataHandle()); + EXPECT_THROWS_AS(h3->openForRead(), fdb5::DaosEntityNotFoundException); + + } + + SECTION("Array read from existing pool and container, with generated OID, reading a range") { + + fdb5::DaosArrayOID oid{3, 6, OC_S1}; + fdb5::DaosArrayName na{pool_name, cont_name, oid}; + std::unique_ptr h(na.dataHandle()); + h->openForWrite(len); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + EXPECT(na.size() == len); + + /// existing generated OID + fdb5::DaosArrayName na_read{pool_name, cont_name, na.OID()}; + std::unique_ptr h2(na_read.dataHandle()); + long skip_bytes = 10; + Length t = h2->openForRead(); + std::vector read_data((size_t)(t - eckit::Length(skip_bytes)), 0); + { + eckit::AutoClose closer(*h2); + EXPECT(h2->size() >= skip_bytes); + EXPECT(eckit::Length(read_data.size()) >= (t - eckit::Length(skip_bytes))); + std::memset(&read_data[0], 0, read_data.size()); + h2->seek(skip_bytes); + res = h2->read(&read_data[0], t - eckit::Length(skip_bytes)); + } + EXPECT(res == len - eckit::Length(skip_bytes)); + EXPECT(std::memcmp(data + skip_bytes, &read_data[0], sizeof(data) - skip_bytes) == 0); + + } + + SECTION("Array read from existing pool and container, with ungenerated OID") { + + fdb5::DaosArrayOID oid{3, 7, OC_S1}; + fdb5::DaosArrayName na{pool_name, cont_name, oid}; + std::unique_ptr h(na.dataHandle()); + h->openForWrite(len); + { + eckit::AutoClose closer(*h); + h->write(data, len); + } + EXPECT(na.size() == len); + + /// existing ungenerated OID + fdb5::DaosArrayName na_read{pool_name, cont_name, oid}; + std::unique_ptr h2(na_read.dataHandle()); + Length t = h2->openForRead(); + std::vector read_data((size_t) t, 0); + { + eckit::AutoClose closer(*h2); + EXPECT(eckit::Length(read_data.size()) >= h2->size()); + std::memset(&read_data[0], 0, read_data.size()); + res = h2->read(&read_data[0], h2->size()); + } + EXPECT(res == len); + EXPECT(std::memcmp(data, &read_data[0], sizeof(data)) == 0); + + /// non-existing ungenerated OID + fdb5::DaosArrayOID oid_ne{3, 8, OC_S1}; + fdb5::DaosArrayName na_read_ne{pool_name, cont_name, oid_ne}; + std::unique_ptr h3(na_read_ne.dataHandle()); + EXPECT_THROWS_AS(h3->openForRead(), fdb5::DaosEntityNotFoundException); + + } + + SECTION("Array read from existing pool and non-existing container") { + + fdb5::DaosArrayOID oid{3, 9, OC_S1}; + fdb5::DaosArrayName na{pool_name, "new_cont_4", oid}; + std::unique_ptr h(na.dataHandle()); + EXPECT_THROWS_AS(h->openForRead(), fdb5::DaosEntityNotFoundException); + EXPECT_NOT(fdb5::DaosName(pool_name, "new_cont_4").exists()); + + } + + /// @todo: test analogous KeyValue workflows + +} + +} // namespace test +} // namespace fdb + +int main(int argc, char **argv) +{ + return run_tests ( argc, argv ); +} diff --git a/tests/fdb/daos/test_daos_store.cc b/tests/fdb/daos/test_daos_store.cc new file mode 100644 index 000000000..aac0464f6 --- /dev/null +++ b/tests/fdb/daos/test_daos_store.cc @@ -0,0 +1,582 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include +#include + +#include "eckit/config/Resource.h" +#include "eckit/testing/Test.h" +#include "eckit/filesystem/URI.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +#include "eckit/filesystem/TmpDir.h" +#include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/config/YAMLConfiguration.h" + +#include "metkit/mars/MarsRequest.h" + +#include "fdb5/fdb5_config.h" +#include "fdb5/config/Config.h" +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" + +#include "fdb5/toc/TocCatalogueWriter.h" +#include "fdb5/toc/TocCatalogueReader.h" + +#include "fdb5/daos/DaosSession.h" +#include "fdb5/daos/DaosPool.h" +#include "fdb5/daos/DaosArrayPartHandle.h" + +#include "fdb5/daos/DaosStore.h" +#include "fdb5/daos/DaosFieldLocation.h" +#include "fdb5/daos/DaosException.h" + +using namespace eckit::testing; +using namespace eckit; + +namespace { + void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); + }; +} + +#ifdef fdb5_HAVE_DUMMY_DAOS +eckit::TmpDir& tmp_dummy_daos_root() { + static eckit::TmpDir d{}; + return d; +} +#endif + +// temporary schema,spaces,root files common to all DAOS Store tests + +eckit::TmpFile& schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::PathName& store_tests_tmp_root() { + static eckit::PathName sd("./daos_store_tests_fdb_root"); + return sd; +} + +namespace fdb { +namespace test { + +CASE( "Setup" ) { + +#ifdef fdb5_HAVE_DUMMY_DAOS + tmp_dummy_daos_root().mkdir(); + ::setenv("DUMMY_DAOS_DATA_ROOT", tmp_dummy_daos_root().path().c_str(), 1); +#endif + + // ensure fdb root directory exists. If not, then that root is + // registered as non existing and Store tests fail. + if (store_tests_tmp_root().exists()) deldir(store_tests_tmp_root()); + store_tests_tmp_root().mkdir(); + ::setenv("FDB_ROOT_DIRECTORY", store_tests_tmp_root().path().c_str(), 1); + + // prepare schema for tests involving DaosStore + + std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; + + std::unique_ptr hs(schema_file().fileHandle()); + hs->openForWrite(schema_str.size()); + { + eckit::AutoClose closer(*hs); + hs->write(schema_str.data(), schema_str.size()); + } + + // this is necessary to avoid ~fdb/etc/fdb/schema being used where + // LibFdb5::instance().defaultConfig().schema() is called + // due to no specified schema file (e.g. in Key::registry()) + ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); + +} + +CASE("DaosStore tests") { + + // test parameters + + int container_oids_per_alloc = 1000; +#if defined(fdb5_HAVE_DAOS_ADMIN) || defined(fdb5_HAVE_DUMMY_DAOS) + std::string pool_name{"fdb_pool"}; +#else + std::string pool_name; + pool_name = eckit::Resource( + "fdbDaosTestPool;$FDB_DAOS_TEST_POOL", pool_name + ); + EXPECT(pool_name.length() > 0); +#endif + + // bootstrap daos + + fdb5::UUID pool_uuid; + { + fdb5::DaosManager::instance().configure( + eckit::LocalConfiguration(YAMLConfiguration( + "container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + )) + ); + fdb5::DaosSession s{}; +#ifdef fdb5_HAVE_DAOS_ADMIN + fdb5::DaosPool& pool = s.createPool(pool_name); +#else + #ifdef fdb5_HAVE_DUMMY_DAOS + std::string pool_uuid_str{"00000000-0000-0000-0000-000000000003"}; + (tmp_dummy_daos_root() / pool_uuid_str).mkdir(); + ::symlink( + (tmp_dummy_daos_root() / pool_uuid_str).path().c_str(), + (tmp_dummy_daos_root() / pool_name).path().c_str() + ); + #endif + fdb5::DaosPool& pool = s.getPool(pool_name); +#endif + pool_uuid = pool.uuid(); + } + + SECTION("archive and retrieve") { + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" + "daos:\n" + " store:\n" + " pool: " + pool_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + fdb5::Schema schema{schema_file()}; + + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}, schema.registry()); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + + char data[] = "test"; + + // archive + + /// DaosManager is configured with client config from the file + fdb5::DaosStore dstore{schema, db_key, config}; + fdb5::Store& store = dstore; + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + /// @todo: two cont create with label happen here + /// @todo: again, daos_fini happening before cont and pool close + + dstore.flush(); // not necessary for DAOS store + + // retrieve + fdb5::Field field(std::move(loc), std::time(nullptr)); + std::cout << "Read location: " << field.location() << std::endl; + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + /// @todo: again, daos_fini happening before + + // remove + fdb5::DaosName field_name{field.location().uri()}; + fdb5::DaosName store_name{field_name.poolName(), field_name.containerName()}; + eckit::URI store_uri(store_name.URI()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT_NOT(store_name.exists()); + + /// @todo: test that when I write multiple things in the same pool, things work as expected + + /// @todo: check that the URI is properly produced + + /// @todo: assert a new DaosSession shows newly configured container_oids_per_alloc + + } + + SECTION("with POSIX Catalogue") { + + // FDB configuration + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "daos:\n" + " store:\n" + " pool: " + pool_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}, schema.registry()); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}, schema.registry()); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}, schema.registry()); + + // store data + + char data[] = "test"; + + fdb5::DaosStore dstore{schema, db_key, config}; + fdb5::Store& store = static_cast(dstore); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + /// @todo: there are two cont create with label here + /// @todo: again, daos_fini happening before cont and pool close + + // index data + + { + /// @todo: could have a unique ptr here, might not need a static cast + fdb5::TocCatalogueWriter tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + cat.deselectIndex(); + cat.selectIndex(index_key); + //const fdb5::Index& idx = tcat.currentIndex(); + static_cast(tcat).archive(field_key, std::move(loc)); + + /// flush store before flushing catalogue + dstore.flush(); // not necessary if using a DAOS store + } + + // find data + + fdb5::Field field; + { + fdb5::TocCatalogueReader tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + cat.selectIndex(index_key); + static_cast(tcat).retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // remove data + + fdb5::DaosName field_name{field.location().uri()}; + fdb5::DaosName store_name{field_name.poolName(), field_name.containerName()}; + eckit::URI store_uri(store_name.URI()); + std::ostream out(std::cout.rdbuf()); + store.remove(store_uri, out, out, false); + EXPECT(field_name.exists()); + store.remove(store_uri, out, out, true); + EXPECT_NOT(field_name.exists()); + EXPECT_NOT(store_name.exists()); + + // deindex data + + { + fdb5::TocCatalogueWriter tcat{db_key, config}; + fdb5::Catalogue& cat = static_cast(tcat); + metkit::mars::MarsRequest r = db_key.request("retrieve"); + std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + cat.visitEntries(*wv, store, false); + } + + /// @todo: again, daos_fini happening before + } + + SECTION("VIA FDB API") { + + // FDB configuration + + int container_oids_per_alloc_small = 100; + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: toc\n" + "store: daos\n" + "daos:\n" + " store:\n" + " pool: " + pool_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc_small) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + + // initialise store + + fdb5::FDB fdb(config); + + // check store is empty + + size_t count; + fdb5::ListElement info; + + /// @todo: here, DaosManager is being configured with DAOS client config passed to FDB instance constructor. + // It happens in EntryVisitMechanism::visit when calling DB::open. Is this OK, or should this configuring + // rather happen as part of transforming a FieldLocation into a DataHandle? It is probably OK. One thing + // is to configure the DAOS client and the other thing is to initialise it. + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // store data + + char data[] = "test"; + + /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times below. + // Should this be avoided? + fdb.archive(request_key, data, sizeof(data)); + + fdb.flush(); + + // retrieve data + + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // wipe data + + fdb5::WipeElement elem; + + // dry run attempt to wipe with too specific request + + auto wipeObject = fdb.wipe(full_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count == 0); + + // dry run wipe index and store unit + wipeObject = fdb.wipe(index_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + + // dry run wipe database + wipeObject = fdb.wipe(db_req); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + + // ensure field still exists + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // attempt to wipe with too specific request + wipeObject = fdb.wipe(full_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count == 0); + /// @todo: really needed? + fdb.flush(); + + // wipe index and store unit (and DB container as there is only one index) + wipeObject = fdb.wipe(index_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) count++; + EXPECT(count == 0); + + /// @todo: ensure index and corresponding container do not exist + + /// @todo: archive two fields on two separate indexes, remove one index, and finally + /// ensure DB still exists with one index + + /// @todo: ensure new DaosSession has updated daos client config + + } + + /// @todo: if doing what's in this section at the end of the previous section reusing the same FDB object, + // archive() fails as it expects a toc file to exist, but it has been removed by previous wipe + SECTION("FDB API RE-STORE AND WIPE DB") { + + // FDB configuration + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: toc\n" + "store: daos\n" + "daos:\n" + " store:\n" + " pool: " + pool_name + "\n" + " client:\n" + " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + }; + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "1"}, {"b", "2"}}); + fdb5::Key index_key({{"a", "1"}, {"b", "2"}, {"c", "3"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + + // initialise store + + fdb5::FDB fdb(config); + + // store again + + char data[] = "test"; + + fdb.archive(request_key, data, sizeof(data)); + + fdb.flush(); + + size_t count; + + // wipe all database + + fdb5::WipeElement elem; + auto wipeObject = fdb.wipe(db_req, true); + count = 0; + while (wipeObject.next(elem)) count++; + EXPECT(count > 0); + /// @todo: really needed? + fdb.flush(); + + // ensure field does not exist + + fdb5::ListElement info; + auto listObject = fdb.list(full_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 0); + + /// @todo: ensure DB and corresponding cont do not exist + + } + + // teardown daos + +#ifdef fdb5_HAVE_DAOS_ADMIN + /// AutoPoolDestroy is not possible here because the pool is + /// created above with an ephemeral session + fdb5::DaosSession().destroyPool(pool_uuid); +#else + for (auto& c : fdb5::DaosSession().getPool(pool_uuid).listContainers()) + if (c == "1:2") + fdb5::DaosSession().getPool(pool_uuid).destroyContainer(c); +#endif + +} + +} // namespace test +} // namespace fdb + +int main(int argc, char **argv) +{ + return run_tests ( argc, argv ); +} diff --git a/tests/fdb/daos/test_dummy_daos_write_read.cc b/tests/fdb/daos/test_dummy_daos_write_read.cc new file mode 100644 index 000000000..33d4f44ac --- /dev/null +++ b/tests/fdb/daos/test_dummy_daos_write_read.cc @@ -0,0 +1,410 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include +#include +#include +#include +#include + +#include "eckit/testing/Test.h" +#include "eckit/filesystem/TmpDir.h" + +#include "daos.h" +#include "dummy_daos.h" +#include "daos/tests_lib.h" + +using namespace eckit::testing; +using namespace eckit; + +eckit::TmpDir& tmp_dummy_daos_root() { + static eckit::TmpDir d{}; + return d; +} + +namespace fdb { +namespace test { + +//---------------------------------------------------------------------------------------------------------------------- + +CASE( "Setup" ) { + + tmp_dummy_daos_root().mkdir(); + ::setenv("DUMMY_DAOS_DATA_ROOT", tmp_dummy_daos_root().path().c_str(), 1); + +} + +CASE( "dummy_daos_write_then_read" ) { + + int rc; + +#ifdef fdb5_HAVE_DAOS_ADMIN + d_rank_list_t svcl; + svcl.rl_nr = 3; + D_ALLOC_ARRAY(svcl.rl_ranks, svcl.rl_nr); + EXPECT(svcl.rl_ranks != NULL); + + // create a pool with user-defined label + + std::string label = "test_pool"; + daos_prop_t *prop = NULL; + prop = daos_prop_alloc(1); + prop->dpp_entries[0].dpe_type = DAOS_PROP_PO_LABEL; + D_STRNDUP(prop->dpp_entries[0].dpe_str, label.c_str(), DAOS_PROP_LABEL_MAX_LEN); + + uuid_t test_pool_uuid; + rc = dmg_pool_create(NULL, geteuid(), getegid(), NULL, NULL, 10ULL << 30, 40ULL << 30, prop, &svcl, test_pool_uuid); + EXPECT(rc == 0); + char test_pool_uuid_str[37] = ""; + uuid_unparse(test_pool_uuid, test_pool_uuid_str); + EXPECT((dummy_daos_root() / test_pool_uuid_str).exists()); + EXPECT((dummy_daos_root() / label).exists()); + + daos_prop_free(prop); + + // create a pool without label + + uuid_t pool_uuid; + rc = dmg_pool_create(NULL, geteuid(), getegid(), NULL, NULL, 10ULL << 30, 40ULL << 30, NULL, &svcl, pool_uuid); + EXPECT(rc == 0); + char uuid_str[37] = ""; + uuid_unparse(pool_uuid, uuid_str); + std::string pool(uuid_str); + eckit::PathName pool_path = dummy_daos_root() / pool; + EXPECT(pool_path.exists()); +#else + std::string pool{"00000000-0000-0000-0000-000000000000"}; + eckit::PathName pool_path = dummy_daos_root() / pool; + pool_path.mkdir(); +#endif + + daos_init(); + + daos_handle_t poh; + + // connect to the pool, create and open a container + + rc = daos_pool_connect(pool.c_str(), NULL, DAOS_PC_RW, &poh, NULL, NULL); + EXPECT(rc == 0); + EXPECT(dummy_daos_get_handle_path(poh) == pool_path); + + // create, open and close a container with auto-generated uuid + + uuid_t cont_uuid = {0}; + rc = daos_cont_create(poh, &cont_uuid, NULL, NULL); + EXPECT(rc == 0); + char cont_uuid_label[37] = ""; + uuid_unparse(cont_uuid, cont_uuid_label); + EXPECT((dummy_daos_get_handle_path(poh) / cont_uuid_label).exists()); + EXPECT((dummy_daos_get_handle_path(poh) / std::string("__dummy_daos_uuid_") + cont_uuid_label).exists()); + + daos_handle_t coh; + + rc = daos_cont_open(poh, cont_uuid_label, DAOS_COO_RW, &coh, NULL, NULL); + EXPECT(rc == 0); + EXPECT(dummy_daos_get_handle_path(coh) == pool_path / cont_uuid_label); + + rc = daos_cont_close(coh, NULL); + EXPECT(rc == 0); + + daos_size_t ncont = 1; + struct daos_pool_cont_info cbuf[ncont]; + rc = daos_pool_list_cont(poh, &ncont, cbuf, NULL); + EXPECT(rc == 0); + EXPECT(ncont == 1); + EXPECT(uuid_compare(cbuf[0].pci_uuid, cont_uuid) == 0); + + rc = daos_cont_destroy(poh, cont_uuid_label, 1, NULL); + EXPECT(rc == 0); + EXPECT(!(dummy_daos_get_handle_path(poh) / cont_uuid_label).exists()); + EXPECT(!(dummy_daos_get_handle_path(poh) / std::string("__dummy_daos_uuid_") + cont_uuid_label).exists()); + + // create and open a container with user-defined label + + std::string cont = "b"; + + uuid_t cont_uuid2 = {0}; + rc = daos_cont_create_with_label(poh, cont.c_str(), NULL, &cont_uuid2, NULL); + EXPECT(rc == 0); + EXPECT((dummy_daos_get_handle_path(poh) / cont).exists()); + char cont_uuid2_str[37] = ""; + uuid_unparse(cont_uuid2, cont_uuid2_str); + EXPECT((dummy_daos_get_handle_path(poh) / cont_uuid2_str).exists()); + + daos_size_t ncont2 = 1; + struct daos_pool_cont_info cbuf2[ncont2]; + rc = daos_pool_list_cont(poh, &ncont2, cbuf2, NULL); + EXPECT(rc == 0); + EXPECT(ncont == 1); + EXPECT(strcmp(cbuf2[0].pci_label, cont.c_str()) == 0); + EXPECT(uuid_compare(cbuf2[0].pci_uuid, cont_uuid2) == 0); + + rc = daos_cont_open(poh, cont.c_str(), DAOS_COO_RW, &coh, NULL, NULL); + EXPECT(rc == 0); + EXPECT(dummy_daos_get_handle_path(coh) == pool_path / cont_uuid2_str); + + daos_size_t size; + daos_size_t oid_alloc_size = 1; + + // Key-Value + + daos_obj_id_t oid_kv; + rc = daos_cont_alloc_oids(coh, oid_alloc_size, &oid_kv.lo, NULL); + EXPECT(rc == 0); + //EXPECT(oid_kv.lo == ((((uint64_t) getpid()) << 48) | (uint64_t) 0)); + + rc = daos_obj_generate_oid(coh, &oid_kv, DAOS_OT_KV_HASHED, OC_S1, 0, 0); + EXPECT(rc == 0); + + daos_handle_t oh_kv; + rc = daos_kv_open(coh, oid_kv, DAOS_OO_RW, &oh_kv, NULL); + EXPECT(rc == 0); + std::stringstream os_kv; + os_kv << std::setw(16) << std::setfill('0') << std::hex << oid_kv.hi; + os_kv << "."; + os_kv << std::setw(16) << std::setfill('0') << std::hex << oid_kv.lo; + EXPECT((dummy_daos_get_handle_path(coh) / os_kv.str()).exists()); + + std::string key = "key"; + std::string value = "value"; + + rc = daos_kv_get(oh_kv, DAOS_TX_NONE, 0, key.c_str(), &size, NULL, NULL); + EXPECT(rc == 0); + EXPECT(size == (daos_size_t) 0); + + rc = daos_kv_put(oh_kv, DAOS_TX_NONE, 0, key.c_str(), std::strlen(value.c_str()), value.c_str(), NULL); + EXPECT(rc == 0); + EXPECT((dummy_daos_get_handle_path(oh_kv) / key).exists()); + + char kv_get_buf[100] = ""; + rc = daos_kv_get(oh_kv, DAOS_TX_NONE, 0, key.c_str(), &size, NULL, NULL); + EXPECT(rc == 0); + EXPECT(size == (daos_size_t) std::strlen(value.c_str())); + rc = daos_kv_get(oh_kv, DAOS_TX_NONE, 0, key.c_str(), &size, kv_get_buf, NULL); + EXPECT(rc == 0); + EXPECT(size == (daos_size_t) std::strlen(value.c_str())); + EXPECT(std::strlen(kv_get_buf) == std::strlen(value.c_str())); + EXPECT(std::string(kv_get_buf) == value); + + std::string key2 = "key2"; + rc = daos_kv_put(oh_kv, DAOS_TX_NONE, 0, key2.c_str(), std::strlen(value.c_str()), value.c_str(), NULL); + EXPECT(rc == 0); + EXPECT((dummy_daos_get_handle_path(oh_kv) / key2).exists()); + + /// @todo: proper memory management + int max_keys_per_rpc = 10; + daos_key_desc_t key_sizes[max_keys_per_rpc]; + d_sg_list_t sgl_kv_list; + d_iov_t iov_kv_list; + char *list_buf; + int bufsize = 1024; + list_buf = (char*) malloc(bufsize); + d_iov_set(&iov_kv_list, list_buf, bufsize); + sgl_kv_list.sg_nr = 1; + sgl_kv_list.sg_nr_out = 0; + sgl_kv_list.sg_iovs = &iov_kv_list; + daos_anchor_t listing_status = DAOS_ANCHOR_INIT; + std::vector listed_keys; + while (!daos_anchor_is_eof(&listing_status)) { + uint32_t nkeys_found = max_keys_per_rpc; + int rc; + memset(list_buf, 0, bufsize); + rc = daos_kv_list(oh_kv, DAOS_TX_NONE, &nkeys_found, key_sizes, &sgl_kv_list, &listing_status, NULL); + EXPECT(rc == 0); + size_t key_start = 0; + for (int i = 0; i < nkeys_found; i++) { + listed_keys.push_back(std::string(list_buf + key_start, key_sizes[i].kd_key_len)); + key_start += key_sizes[i].kd_key_len; + } + } + EXPECT(listed_keys.size() == 2); + EXPECT(listed_keys[0] == key); + EXPECT(listed_keys[1] == key2); + + rc = daos_kv_remove(oh_kv, DAOS_TX_NONE, 0, key.c_str(), NULL); + EXPECT(rc == 0); + EXPECT_NOT((dummy_daos_get_handle_path(oh_kv) / key).exists()); + + daos_obj_close(oh_kv, NULL); + EXPECT(rc == 0); + + // Array write + + daos_obj_id_t oid; + rc = daos_cont_alloc_oids(coh, oid_alloc_size, &oid.lo, NULL); + EXPECT(rc == 0); + //EXPECT(oid.lo == ((((uint64_t) getpid()) << 48) | (uint64_t) 1)); + + rc = daos_array_generate_oid(coh, &oid, true, OC_S1, 0, 0); + EXPECT(rc == 0); + + daos_handle_t oh; + rc = daos_array_create(coh, oid, DAOS_TX_NONE, 1, 1048576, &oh, NULL); + EXPECT(rc == 0); + std::stringstream os; + os << std::setw(16) << std::setfill('0') << std::hex << oid.hi; + os << "."; + os << std::setw(16) << std::setfill('0') << std::hex << oid.lo; + EXPECT((dummy_daos_get_handle_path(coh) / os.str()).exists()); + + char data[] = "test"; + + daos_array_iod_t iod; + daos_range_t rg; + + d_sg_list_t sgl; + d_iov_t iov; + + iod.arr_nr = 1; + rg.rg_len = (daos_size_t) sizeof(data); + rg.rg_idx = (daos_off_t) 0; + iod.arr_rgs = &rg; + + sgl.sg_nr = 1; + d_iov_set(&iov, data, (size_t) sizeof(data)); + sgl.sg_iovs = &iov; + + rc = daos_array_write(oh, DAOS_TX_NONE, &iod, &sgl, NULL); + EXPECT(rc == 0); + EXPECT(dummy_daos_get_handle_path(oh).exists()); + + daos_array_close(oh, NULL); + EXPECT(rc == 0); + + // Array read + + daos_size_t cell_size, csize; + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW, &cell_size, &csize, &oh, NULL); + EXPECT(rc == 0); + + char *data_read; + + daos_size_t array_size; + rc = daos_array_get_size(oh, DAOS_TX_NONE, &array_size, NULL); + EXPECT(rc == 0); + EXPECT(array_size == rg.rg_len); + + data_read = (char *) malloc(sizeof(char) * ((size_t) array_size)); + + iod.arr_nr = 1; + rg.rg_len = array_size; + rg.rg_idx = (daos_off_t) 0; + iod.arr_rgs = &rg; + + sgl.sg_nr = 1; + d_iov_set(&iov, data_read, (size_t) array_size); + sgl.sg_iovs = &iov; + + rc = daos_array_read(oh, DAOS_TX_NONE, &iod, &sgl, NULL); + EXPECT(rc == 0); + EXPECT(std::memcmp(data, data_read, sizeof(data)) == 0); + + free(data_read); + + rc = daos_array_close(oh, NULL); + EXPECT(rc == 0); + + // container list OIDs + + daos_epoch_t e; + rc = daos_cont_create_snap_opt( + coh, &e, NULL, (enum daos_snapshot_opts)(DAOS_SNAP_OPT_CR | DAOS_SNAP_OPT_OIT), NULL + ); + EXPECT(rc == 0); + std::stringstream os_epoch; + os_epoch << std::setw(16) << std::setfill('0') << std::hex << e; + EXPECT((dummy_daos_get_handle_path(coh) / os_epoch.str() + ".snap").exists()); + + daos_handle_t oith; + rc = daos_oit_open(coh, e, &oith, NULL); + EXPECT(rc == 0); + + daos_anchor_t anchor = DAOS_ANCHOR_INIT; + int max_oids_per_rpc = 10; + daos_obj_id_t oid_batch[max_oids_per_rpc]; + std::vector oids; + while (!daos_anchor_is_eof(&anchor)) { + uint32_t oids_nr = max_oids_per_rpc; + rc = daos_oit_list(oith, oid_batch, &oids_nr, &anchor, NULL); + EXPECT(rc == 0); + for (int i = 0; i < oids_nr; i++) oids.push_back(oid_batch[i]); + } + EXPECT(oids.size() == 2); + EXPECT(std::memcmp(&oids[0], &oid_kv, sizeof(daos_obj_id_t)) == 0); + EXPECT(std::memcmp(&oids[1], &oid, sizeof(daos_obj_id_t)) == 0); + + rc = daos_oit_close(oith, NULL); + EXPECT(rc == 0); + + daos_epoch_range_t epr{e, e}; + rc = daos_cont_destroy_snap(coh, epr, NULL); + EXPECT(rc == 0); + EXPECT_NOT((dummy_daos_get_handle_path(coh) / os_epoch.str() + ".snap").exists()); + + // close container and pool, finalize DAOS client + + rc = daos_kv_open(coh, oid_kv, DAOS_OO_RW, &oh_kv, NULL); + EXPECT(rc == 0); + + rc = daos_kv_destroy(oh_kv, DAOS_TX_NONE, NULL); + EXPECT(rc == 0); + + daos_obj_close(oh_kv, NULL); + EXPECT(rc == 0); + + rc = daos_array_open(coh, oid, DAOS_TX_NONE, DAOS_OO_RW, &cell_size, &csize, &oh, NULL); + EXPECT(rc == 0); + + rc = daos_array_destroy(oh, DAOS_TX_NONE, NULL); + EXPECT(rc == 0); + EXPECT_NOT(dummy_daos_get_handle_path(oh).exists()); + + rc = daos_array_close(oh, NULL); + EXPECT(rc == 0); + + rc = daos_cont_close(coh, NULL); + EXPECT(rc == 0); + + rc = daos_cont_destroy(poh, cont.c_str(), 1, NULL); + EXPECT(rc == 0); + EXPECT(!(dummy_daos_get_handle_path(poh) / cont).exists()); + EXPECT(!(dummy_daos_get_handle_path(poh) / cont_uuid2_str).exists()); + + rc = daos_pool_disconnect(poh, NULL); + EXPECT(rc == 0); + + daos_fini(); + +#ifdef fdb5_HAVE_DAOS_ADMIN + // destroy the pools + + rc = dmg_pool_destroy(NULL, pool_uuid, NULL, 1); + EXPECT(rc == 0); + EXPECT(!pool_path.exists()); + + rc = dmg_pool_destroy(NULL, test_pool_uuid, NULL, 1); + EXPECT(rc == 0); + EXPECT(!(dummy_daos_root() / test_pool_uuid_str).exists()); + EXPECT(!(dummy_daos_root() / label).exists()); + + D_FREE(svcl.rl_ranks); +#endif + +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace test +} // namespace fdb + +int main(int argc, char **argv) +{ + return run_tests ( argc, argv ); +} diff --git a/tests/regressions/FDB-241/FDB-241.sh.in b/tests/regressions/FDB-241/FDB-241.sh.in index 8b34cd228..d0b0911c0 100755 --- a/tests/regressions/FDB-241/FDB-241.sh.in +++ b/tests/regressions/FDB-241/FDB-241.sh.in @@ -37,12 +37,12 @@ $fdbwrite 6.odb cat > list < list_fsoifb < list < listLength < shortOut cmp shortOut listLength diff --git a/tests/regressions/FDB-268/FDB-268.sh.in b/tests/regressions/FDB-268/FDB-268.sh.in index f9c4c11a2..420fb7e0d 100755 --- a/tests/regressions/FDB-268/FDB-268.sh.in +++ b/tests/regressions/FDB-268/FDB-268.sh.in @@ -35,9 +35,9 @@ cat > list < listLength < r1 < shortOut cmp shortOut listLength diff --git a/tests/regressions/FDB-271/FDB-271.sh.in b/tests/regressions/FDB-271/FDB-271.sh.in index 48e51193e..6f02485ae 100755 --- a/tests/regressions/FDB-271/FDB-271.sh.in +++ b/tests/regressions/FDB-271/FDB-271.sh.in @@ -47,9 +47,9 @@ cat > old_list < old_listLength < old_r1 <