Update with latest feture/daos_handle and resolve conflicts.

ecmwf · May 15, 2024 · 99d7d7c · 99d7d7c
2 parents 229f9d7 + 02759a5
commit 99d7d7c
Show file tree

Hide file tree

Showing 150 changed files with 2,279 additions and 1,195 deletions.
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
@@ -0,0 +1,26 @@
+name: sync
+
+# Controls when the workflow will run
+on:
+
+  # Trigger the workflow on all pushes
+  push:
+    branches:
+    - '**'
+    tags:
+    - '**'
+
+  # Trigger the workflow when a branch or tag is deleted
+  delete: ~
+
+jobs:
+
+  # Calls a reusable CI workflow to sync the current with a remote repository.
+  #   It will correctly handle addition of any new and removal of existing Git objects.
+  sync:
+    name: sync
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/sync.yml@v2
+    secrets:
+      target_repository: mars/fdb5
+      target_username: ClonedDuck
+      target_token: ${{ secrets.BITBUCKET_PAT }}
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-5.11.29
+5.12.1
diff --git a/src/dummy_daos/daos/tests_lib.cc b/src/dummy_daos/daos/tests_lib.cc
@@ -209,16 +209,14 @@ int dmg_pool_destroy(const char *dmg_config_file,
     ///        specified pool UUID. None will match if the pool was not labelled.
 
     for (auto& f : files) {
-        if (f.exists()) {
-            try {
+        try {
 
-                if (f.isLink() && f.realName().baseName() == pool_path.baseName()) f.unlink();
+            if (f.isLink() && f.realName().baseName() == pool_path.baseName()) f.unlink();
 
-            } catch (eckit::FailedSystemCall& e) {
+        } catch (eckit::FailedSystemCall& e) {
 
-                if (f.exists()) throw;
+            if (f.exists()) throw;
 
-            }
         }
     }
 

diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt
@@ -32,6 +32,8 @@ list( APPEND fdb5_srcs
     api/SelectFDB.h
 
     api/helpers/APIIterator.h
+    api/helpers/AxesIterator.cc
+    api/helpers/AxesIterator.h
     api/helpers/ControlIterator.cc
     api/helpers/ControlIterator.h
     api/helpers/FDBToolRequest.cc
@@ -50,6 +52,8 @@ list( APPEND fdb5_srcs
     api/local/QueryVisitor.h
     api/local/QueueStringLogTarget.h
     api/local/ListVisitor.h
+    api/local/AxesVisitor.cc
+    api/local/AxesVisitor.h
     api/local/ControlVisitor.cc
     api/local/ControlVisitor.h
     api/local/DumpVisitor.h
@@ -77,6 +81,8 @@ list( APPEND fdb5_srcs
     database/Catalogue.h
     database/DB.cc
     database/DB.h
+    database/DatabaseNotFoundException.cc
+    database/DatabaseNotFoundException.h
     database/DataStats.cc
     database/DataStats.h
     database/DbStats.cc
@@ -413,6 +419,8 @@ if( HAVE_DAOSFDB )
         daos/DaosArrayPartHandle.cc
         daos/DaosLazyFieldLocation.h 
         daos/DaosLazyFieldLocation.cc
+        daos/UUID.h
+        daos/UUID.cc
     )
 else()
     set( DAOS_LIBRARIES "" )
@@ -470,6 +478,7 @@ if(HAVE_FDB_BUILD_TOOLS)
 endif()
 
 list( APPEND fdb5_tools
+    fdb-axes
     fdb-write
     fdb-copy
     fdb-dump
@@ -503,7 +512,7 @@ foreach( _tool ${fdb5_tools} )
   ecbuild_add_executable( TARGET    ${_tool}
                           CONDITION HAVE_FDB_BUILD_TOOLS
                           SOURCES   tools/${_tool}.cc
-                          INCLUDES  ${ECCODES_INCLUDE_DIRS} ${DAOS_INCLUDE_DIRS}  # Please don't remove me, I am needed
+                          INCLUDES  ${ECCODES_INCLUDE_DIRS}  # Please don't remove me, I am needed
                           LIBS      fdb5 )
 
 endforeach()

diff --git a/src/fdb5/LibFdb5.cc b/src/fdb5/LibFdb5.cc
@@ -40,10 +40,10 @@ LibFdb5& LibFdb5::instance() {
     return libfdb;
 }
 
-const Config& LibFdb5::defaultConfig() { 
+const Config& LibFdb5::defaultConfig(const eckit::Configuration& userConfig) { 
     if(!config_) {
         Config cfg;
-        config_.reset( new Config( std::move(cfg.expandConfig()) ) );
+        config_.reset( new Config( std::move(cfg.expandConfig()), userConfig ) );
     }
     return *config_;
 }
@@ -81,7 +81,7 @@ static unsigned getUserEnvRemoteProtocol() {
     static unsigned fdbRemoteProtocolVersion =
         eckit::Resource<unsigned>("fdbRemoteProtocolVersion;$FDB5_REMOTE_PROTOCOL_VERSION", 0);
     if (fdbRemoteProtocolVersion) {
-        eckit::Log::debug() << "fdbRemoteProtocolVersion overidde to version: " << fdbRemoteProtocolVersion
+        LOG_DEBUG_LIB(LibFdb5) << "fdbRemoteProtocolVersion overidde to version: " << fdbRemoteProtocolVersion
                             << std::endl;
     }
     return 0;  // no version override

diff --git a/src/fdb5/LibFdb5.h b/src/fdb5/LibFdb5.h
@@ -71,7 +71,7 @@ class LibFdb5 : public eckit::system::Library {
     RemoteProtocolVersion remoteProtocolVersion() const;
 
     /// Returns the default configuration according to the rules of FDB configuration search
-    const Config& defaultConfig();
+    const Config& defaultConfig(const eckit::Configuration& userConfig = eckit::LocalConfiguration());
 
     bool dontDeregisterFactories() const;
 

diff --git a/src/fdb5/api/DistFDB.cc b/src/fdb5/api/DistFDB.cc
@@ -74,21 +74,21 @@ void DistFDB::archive(const Key& key, const void* data, size_t length) {
 
     std::vector<size_t> laneIndices;
 
-    //Log::debug<LibFdb5>() << "Number of lanes: " << lanes_.size() << std::endl;
-    //Log::debug<LibFdb5>() << "Lane indices: ";
-    //for (const auto& i : laneIndices) Log::debug<LibFdb5>() << i << ", ";
-    //Log::debug<LibFdb5>() << std::endl;
+    //LOG_DEBUG_LIB(LibFdb5) << "Number of lanes: " << lanes_.size() << std::endl;
+    //LOG_DEBUG_LIB(LibFdb5) << "Lane indices: ";
+    //for (const auto& i : laneIndices) LOG_DEBUG_LIB(LibFdb5) << i << ", ";
+    //LOG_DEBUG_LIB(LibFdb5) << std::endl;
 
     hash_.hashOrder(key.keyDict(), laneIndices);
 
-    //Log::debug<LibFdb5>() << "Number of lanes: " << lanes_.size() << std::endl;
-    //Log::debug<LibFdb5>() << "Lane indices: ";
-    //for (const auto& i : laneIndices) Log::debug<LibFdb5>() << i << ", ";
-    //Log::debug<LibFdb5>() << std::endl;
+    //LOG_DEBUG_LIB(LibFdb5) << "Number of lanes: " << lanes_.size() << std::endl;
+    //LOG_DEBUG_LIB(LibFdb5) << "Lane indices: ";
+    //for (const auto& i : laneIndices) LOG_DEBUG_LIB(LibFdb5) << i << ", ";
+    //LOG_DEBUG_LIB(LibFdb5) << std::endl;
 
     // Given an order supplied by the Rendezvous hash, try the FDB in order until
     // one works. n.b. Errors are unacceptable once the FDB is dirty.
-    Log::debug<LibFdb5>() << "Attempting dist FDB archive" << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Attempting dist FDB archive" << std::endl;
 
     decltype(laneIndices)::const_iterator it = laneIndices.begin();
     decltype(laneIndices)::const_iterator end = laneIndices.end();
@@ -180,55 +180,55 @@ auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) ->
 
 
 ListIterator DistFDB::list(const FDBToolRequest& request) {
-    Log::debug<LibFdb5>() << "DistFDB::list() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::list() : " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.list(request);
                          });
 }
 
 ListIterator DistFDB::inspect(const metkit::mars::MarsRequest& request) {
-    Log::debug<LibFdb5>() << "DistFDB::inspect() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::inspect() : " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.inspect(request.request());
                          });
 }
 
 DumpIterator DistFDB::dump(const FDBToolRequest& request, bool simple) {
-    Log::debug<LibFdb5>() << "DistFDB::dump() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::dump() : " << request << std::endl;
     return queryInternal(request,
                          [simple](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.dump(request, simple);
                          });
 }
 
 StatusIterator DistFDB::status(const FDBToolRequest& request) {
-    Log::debug<LibFdb5>() << "DistFDB::status() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::status() : " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.status(request);
     });
 }
 
 WipeIterator DistFDB::wipe(const FDBToolRequest& request, bool doit, bool porcelain, bool unsafeWipeAll) {
-    Log::debug<LibFdb5>() << "DistFDB::wipe() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::wipe() : " << request << std::endl;
     return queryInternal(request,
                          [doit, porcelain, unsafeWipeAll](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.wipe(request, doit, porcelain, unsafeWipeAll);
     });
 }
 
 PurgeIterator DistFDB::purge(const FDBToolRequest& request, bool doit, bool porcelain) {
-    Log::debug<LibFdb5>() << "DistFDB::purge() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::purge() : " << request << std::endl;
     return queryInternal(request,
                          [doit, porcelain](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.purge(request, doit, porcelain);
     });
 }
 
 StatsIterator DistFDB::stats(const FDBToolRequest &request) {
-    Log::debug<LibFdb5>() << "DistFDB::stats() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::stats() : " << request << std::endl;
     return queryInternal(request,
                          [](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.stats(request);
@@ -238,7 +238,7 @@ StatsIterator DistFDB::stats(const FDBToolRequest &request) {
 ControlIterator DistFDB::control(const FDBToolRequest& request,
                                  ControlAction action,
                                  ControlIdentifiers identifiers) {
-    Log::debug<LibFdb5>() << "DistFDB::control() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::control() : " << request << std::endl;
     return queryInternal(request,
                          [action, identifiers](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.control(request, action, identifiers);
@@ -247,7 +247,7 @@ ControlIterator DistFDB::control(const FDBToolRequest& request,
 
 
 MoveIterator DistFDB::move(const FDBToolRequest& request, const eckit::URI& dest) {
-    Log::debug<LibFdb5>() << "DistFDB::move() : " << request << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "DistFDB::move() : " << request << std::endl;
     return queryInternal(request,
                          [dest](FDB& fdb, const FDBToolRequest& request) {
                             return fdb.move(request, dest);

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
@@ -29,6 +29,7 @@
 #include "fdb5/database/Key.h"
 #include "fdb5/io/HandleGatherer.h"
 #include "fdb5/message/MessageDecoder.h"
+#include "fdb5/types/Type.h"
 
 // comment out for DAOS runs
 //#include "fdb5/toc/TocSession.h"
@@ -82,7 +83,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h
             ss << "FDB archive - found unexpected message" << std::endl;
             ss << "  user request:"  << std::endl << "    " << request << std::endl;
             ss << "  unexpected message:" << std::endl << "    " << key << std::endl;
-            eckit::Log::debug<LibFdb5>() << ss.str();
+            LOG_DEBUG_LIB(LibFdb5) << ss.str();
             throw eckit::UserError(ss.str(), Here());
         }
         archive(key, msg.data(), msg.length());
@@ -95,7 +96,7 @@ void FDB::archive(const metkit::mars::MarsRequest& request, eckit::DataHandle& h
         for (auto vacantRequest : cube.vacantRequests()) {
             ss << "    " << vacantRequest << std::endl;
         }
-        eckit::Log::debug<LibFdb5>() << ss.str();
+        LOG_DEBUG_LIB(LibFdb5) << ss.str();
         throw eckit::UserError(ss.str(), Here());
     }
 }
@@ -104,24 +105,24 @@ void FDB::archive(const Key& key, const void* data, size_t length) {
     eckit::Timer timer;
     timer.start();
 
-    auto stepunit = key.find("stepunits");
-    if (stepunit != key.end()) {
-        Key k;
-        for (auto it : key) {
-            if (it.first == "step" && stepunit->second.size()>0 && stepunit->second[0]!='h') {
-                // TODO - enable canonical representation of step (as soon as Metkit supports it)
-                std::string canonicalStep = it.second+stepunit->second; // k.registry().lookupType("step").toKey("step", it.second+stepunit->second);
-                k.set(it.first, canonicalStep);
-            } else {
-                if (it.first != "stepunits") {
-                    k.set(it.first, it.second);
-                }
+    // This is the API entrypoint. Keys supplied by the user may not have type registry info attached (so
+    // serialisation won't work properly...)
+    Key keyInternal(key);
+    keyInternal.registry(config().schema().registry());
+
+    // step in archival requests from the model is just an integer. We need to include the stepunit
+    auto stepunit = keyInternal.find("stepunits");
+    if (stepunit != keyInternal.end()) {
+        if (stepunit->second.size()>0 && stepunit->second[0]!='h') {
+            auto step = keyInternal.find("step");
+            if (step != keyInternal.end()) {
+                std::string canonicalStep = keyInternal.registry().lookupType("step").toKey("step", step->second+stepunit->second);
             }
         }
-        internal_->archive(k, data, length);
-    } else {
-        internal_->archive(key, data, length);
+        keyInternal.unset("stepunits");
     }
+
+    internal_->archive(keyInternal, data, length);
     dirty_ = true;
 
     timer.stop();
@@ -139,7 +140,7 @@ bool FDB::sorted(const metkit::mars::MarsRequest &request) {
         eckit::Log::userInfo() << "Using optimise" << std::endl;
     }
 
-    eckit::Log::debug<LibFdb5>() << "fdb5::FDB::retrieve() Sorted? " << sorted << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "fdb5::FDB::retrieve() Sorted? " << sorted << std::endl;
 
     return sorted;
 }
@@ -307,6 +308,16 @@ void FDB::flush() {
     }
 }
 
+IndexAxis FDB::axes(const FDBToolRequest& request, int level) {
+    IndexAxis axes;
+    AxesElement elem;
+    auto it = internal_->axes(request, level);
+    while (it.next(elem)) {
+        axes.merge(elem.axes());
+    }
+    return axes;
+}
+
 bool FDB::dirty() const {
     return dirty_;
 }

diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h
@@ -109,6 +109,9 @@ class FDB {
     ControlIterator control(const FDBToolRequest& request,
                             ControlAction action,
                             ControlIdentifiers identifiers);
+
+    IndexAxis axes(const FDBToolRequest& request, int level=3);
+
     bool enabled(const ControlIdentifier& controlIdentifier) const;
 
     bool dirty() const;

diff --git a/src/fdb5/api/FDBFactory.cc b/src/fdb5/api/FDBFactory.cc
@@ -50,7 +50,7 @@ FDBBase::FDBBase(const Config& config, const std::string& name) :
         controlIdentifiers_ |= ControlIdentifier::Wipe;
     }
 
-    eckit::Log::debug<LibFdb5>() << "FDBBase: " << config << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "FDBBase: " << config << std::endl;
 }
 
 
@@ -114,7 +114,7 @@ std::unique_ptr<FDBBase> FDBFactory::build(const Config& config) {
 
     std::string key = actualConfig.getString("type", "local");
 
-    eckit::Log::debug<LibFdb5>() << "Selecting FDB implementation: " << key << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Selecting FDB implementation: " << key << std::endl;
 
     eckit::AutoLock<eckit::Mutex> lock(mutex_);
 
@@ -127,7 +127,7 @@ std::unique_ptr<FDBBase> FDBFactory::build(const Config& config) {
     }
 
     std::unique_ptr<FDBBase> ret = it->second->make(actualConfig);
-    eckit::Log::debug<LibFdb5>() << "Constructed FDB implementation: " << *ret << std::endl;
+    LOG_DEBUG_LIB(LibFdb5) << "Constructed FDB implementation: " << *ret << std::endl;
     return ret;
 }
 

diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h
@@ -28,7 +28,7 @@
 #include "fdb5/database/DB.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/api/FDBStats.h"
-#include "fdb5/api/helpers/ListIterator.h"
+#include "fdb5/api/helpers/AxesIterator.h"
 #include "fdb5/api/helpers/ListIterator.h"
 #include "fdb5/api/helpers/ControlIterator.h"
 #include "fdb5/api/helpers/DumpIterator.h"
@@ -88,6 +88,8 @@ class FDBBase : private eckit::NonCopyable {
 
     virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0;
 
+    virtual AxesIterator axes(const FDBToolRequest& request, int axes) { NOTIMP; }
+
     // -------------- API management ----------------------------
 
     /// ID used for hashing in the Rendezvous hash. Should be unique amongst those used