Merge branch 'develop' into hotfix/5.13.2

ecmwf · Oct 7, 2024 · a789091 · a789091
2 parents 2fccbdc + 1001d61
commit a789091
Show file tree

Hide file tree

Showing 205 changed files with 2,105 additions and 1,392 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
 .vscode
+build
diff --git a/src/fdb5/LibFdb5.h b/src/fdb5/LibFdb5.h
@@ -20,7 +20,6 @@
 #include "eckit/system/Library.h"
 
 #include "fdb5/database/DB.h"
-#include "fdb5/types/TypesRegistry.h"
 #include "fdb5/api/helpers/Callback.h"
 
 namespace fdb5 {

diff --git a/src/fdb5/api/DistFDB.h b/src/fdb5/api/DistFDB.h
@@ -69,7 +69,7 @@ class DistFDB : public FDBBase {
 
 private: // methods
 
-    virtual void print(std::ostream& s) const override;
+    void print(std::ostream& s) const override;
 
     template <typename QueryFN>
     auto queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> decltype(fn(*(FDB*)(nullptr), request));

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
@@ -20,9 +20,6 @@
 #include "eckit/message/Message.h"
 #include "eckit/message/Reader.h"
 
-#include "eckit/system/Plugin.h"
-#include "eckit/system/LibraryManager.h"
-
 #include "metkit/hypercube/HyperCubePayloaded.h"
 
 #include "fdb5/LibFdb5.h"
@@ -34,6 +31,8 @@
 #include "fdb5/message/MessageDecoder.h"
 #include "fdb5/types/Type.h"
 
+#include <memory>
+
 namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -109,15 +108,15 @@ void FDB::archive(const Key& key, const void* data, size_t length) {
     // This is the API entrypoint. Keys supplied by the user may not have type registry info attached (so
     // serialisation won't work properly...)
     Key keyInternal(key);
-    keyInternal.registry(config().schema().registry());
 
     // step in archival requests from the model is just an integer. We need to include the stepunit
     auto stepunit = keyInternal.find("stepunits");
     if (stepunit != keyInternal.end()) {
-        if (stepunit->second.size()>0 && stepunit->second[0]!='h') {
+        if (stepunit->second.size()>0 && static_cast<char>(tolower(stepunit->second[0])) != 'h') {
             auto step = keyInternal.find("step");
             if (step != keyInternal.end()) {
-                std::string canonicalStep = keyInternal.registry().lookupType("step").toKey("step", step->second+stepunit->second);
+                std::string canonicalStep = config().schema().registry().lookupType("step").toKey(step->second + static_cast<char>(tolower(stepunit->second[0])));
+                keyInternal.set("step", canonicalStep);
             }
         }
         keyInternal.unset("stepunits");
@@ -154,21 +153,20 @@ class ListElementDeduplicator : public metkit::hypercube::Deduplicator<ListEleme
 };
 
 eckit::DataHandle* FDB::read(const eckit::URI& uri) {
-    FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri);
-    return loc->dataHandle();
+    auto location = std::unique_ptr<FieldLocation>(FieldLocationFactory::instance().build(uri.scheme(), uri));
+    return location->dataHandle();
 }
 
 eckit::DataHandle* FDB::read(const std::vector<eckit::URI>& uris, bool sorted) {
     HandleGatherer result(sorted);
 
     for (const eckit::URI& uri : uris) {
-        FieldLocation* loc = FieldLocationFactory::instance().build(uri.scheme(), uri);
-        result.add(loc->dataHandle());
-        delete loc;
+        auto location = std::unique_ptr<FieldLocation>(FieldLocationFactory::instance().build(uri.scheme(), uri));
+        result.add(location->dataHandle());
     }
+
     return result.dataHandle();
 }
-
 
 eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
     eckit::Timer timer;
@@ -303,13 +301,17 @@ void FDB::flush() {
 IndexAxis FDB::axes(const FDBToolRequest& request, int level) {
     IndexAxis axes;
     AxesElement elem;
-    auto it = internal_->axes(request, level);
+    auto it = axesIterator(request, level);
     while (it.next(elem)) {
         axes.merge(elem.axes());
     }
     return axes;
 }
 
+AxesIterator FDB::axesIterator(const FDBToolRequest& request, int level) {
+    return internal_->axesIterator(request, level);
+}
+
 bool FDB::dirty() const {
     return dirty_;
 }

diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h
@@ -33,6 +33,7 @@
 #include "fdb5/api/helpers/StatusIterator.h"
 #include "fdb5/api/helpers/WipeIterator.h"
 #include "fdb5/api/helpers/MoveIterator.h"
+#include "fdb5/api/helpers/AxesIterator.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/api/helpers/Callback.h"
 
@@ -115,6 +116,8 @@ class FDB {
 
     IndexAxis axes(const FDBToolRequest& request, int level=3);
 
+    AxesIterator axesIterator(const FDBToolRequest& request, int level=3);
+
     bool enabled(const ControlIdentifier& controlIdentifier) const;
 
     bool dirty() const;

diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h
@@ -89,7 +89,7 @@ class FDBBase : private eckit::NonCopyable {
 
     virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0;
 
-    virtual AxesIterator axes(const FDBToolRequest& request, int axes) { NOTIMP; }
+    virtual AxesIterator axesIterator(const FDBToolRequest& request, int axes) { NOTIMP; }
 
     void registerArchiveCallback(ArchiveCallback callback) {callback_ = callback;}
 
@@ -185,7 +185,7 @@ class FDBBuilder : public FDBBuilderBase {
 
 private: // methods
 
-    virtual std::unique_ptr<FDBBase> make(const Config& config) const override {
+    std::unique_ptr<FDBBase> make(const Config& config) const override {
         return std::unique_ptr<T>(new T(config, name_));
     }
 };

diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc
@@ -125,8 +125,8 @@ ControlIterator LocalFDB::control(const FDBToolRequest& request,
     return queryInternal<ControlVisitor>(request, action, identifiers);
 }
 
-AxesIterator LocalFDB::axes(const FDBToolRequest& request, int level) {
-    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axes() : " << request << std::endl;
+AxesIterator LocalFDB::axesIterator(const FDBToolRequest& request, int level) {
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axesIterator() : " << request << std::endl;
     return queryInternal<AxesVisitor>(request, config_, level);
 }
 

diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h
@@ -59,7 +59,7 @@ class LocalFDB : public FDBBase {
 
     MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override;
 
-    AxesIterator axes(const FDBToolRequest& request, int axes) override;
+    AxesIterator axesIterator(const FDBToolRequest& request, int axes) override;
 
     void flush() override;
 

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
@@ -968,7 +968,7 @@ class FDBRemoteDataHandle : public DataHandle {
         overallPosition_(0),
         currentBuffer_(0),
         complete_(false) {}
-    virtual bool canSeek() const override { return false; }
+    bool canSeek() const override { return false; }
 
 private: // methods
 

diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
@@ -126,9 +126,9 @@ class RemoteFDB : public FDBBase {
     void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
     long sendArchiveData(uint32_t id, const std::vector<std::pair<Key, eckit::Buffer>>& elements, size_t count);
 
-    virtual void print(std::ostream& s) const override;
+    void print(std::ostream& s) const override;
 
-    virtual FDBStats stats() const override;
+    FDBStats stats() const override;
 
 private: // members
 

diff --git a/src/fdb5/api/SelectFDB.cc b/src/fdb5/api/SelectFDB.cc
@@ -196,6 +196,14 @@ ControlIterator SelectFDB::control(const FDBToolRequest& request,
     });
 }
 
+AxesIterator SelectFDB::axesIterator(const FDBToolRequest& request, int level) {
+    LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::axesIterator() >> " << request << std::endl;
+    return queryInternal(request,
+                         [level](FDB& fdb, const FDBToolRequest& request) {
+                            return fdb.axesIterator(request, level);
+    });
+}
+
 void SelectFDB::flush() {
     for (auto& iter : subFdbs_) {
         FDB& fdb(iter.second);
@@ -208,7 +216,7 @@ void SelectFDB::print(std::ostream &s) const {
     s << "SelectFDB()";
 }
 
-bool SelectFDB::matches(const Key &key, const SelectMap &select, bool requireMissing) const {
+bool SelectFDB::matches(const Key& key, const SelectMap &select, bool requireMissing) const {
 
     for (const auto& kv : select) {
 

diff --git a/src/fdb5/api/SelectFDB.h b/src/fdb5/api/SelectFDB.h
@@ -68,6 +68,8 @@ class SelectFDB : public FDBBase {
 
     MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override { NOTIMP; }
 
+    AxesIterator axesIterator(const FDBToolRequest& request, int level) override;
+
     void flush() override;
 
 private: // methods

diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc
@@ -12,6 +12,7 @@
 #include "eckit/io/FileDescHandle.h"
 #include "eckit/message/Message.h"
 #include "eckit/runtime/Main.h"
+#include "eckit/config/YAMLConfiguration.h"
 
 #include "metkit/mars/MarsRequest.h"
 #include "metkit/mars/MarsExpension.h"
@@ -331,6 +332,16 @@ int fdb_new_handle(fdb_handle_t** fdb) {
     });
 }
 
+int fdb_new_handle_from_yaml(fdb_handle_t** fdb, const char* system_config, const char* user_config) {
+    return wrapApiFunction([fdb, system_config, user_config] {
+        Config cfg{YAMLConfiguration(std::string(system_config)), YAMLConfiguration(std::string(user_config))};
+        cfg.set("configSource", "yaml");
+        cfg.expandConfig();
+        *fdb = new fdb_handle_t(cfg);
+    });
+
+}
+
 int fdb_archive(fdb_handle_t* fdb, fdb_key_t* key, const char* data, size_t length) {
     return wrapApiFunction([fdb, key, data, length] {
         ASSERT(fdb);

diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h
@@ -310,6 +310,14 @@ typedef struct fdb_handle_t fdb_handle_t;
  */
 int fdb_new_handle(fdb_handle_t** fdb);
 
+/** Creates a FDB instance from a YAML configuration.
+ * \param fdb FDB instance. Returned instance must be deleted using #fdb_delete_handle.
+ * \param system_config Override the system config with this YAML string
+ * \param user_config Supply user level config with this YAML string
+ * \returns Return code (#FdbErrorValues)
+ */
+int fdb_new_handle_from_yaml(fdb_handle_t** fdb, const char* system_config, const char* user_config);
+
 /** Archives binary data to a FDB instance.
  * \warning this is a low-level API. The provided key and the corresponding data are not checked for consistency
  * \param fdb FDB instance.

diff --git a/src/fdb5/api/helpers/APIIterator.h b/src/fdb5/api/helpers/APIIterator.h
@@ -83,9 +83,9 @@ class APIAggregateIterator : public APIIteratorBase<ValueType> {
     APIAggregateIterator(std::queue<APIIterator<ValueType>>&& iterators) :
         iterators_(std::move(iterators)) {}
 
-    virtual ~APIAggregateIterator() override {}
+    ~APIAggregateIterator() override {}
 
-    virtual bool next(ValueType& elem) override {
+    bool next(ValueType& elem) override {
 
         while (!iterators_.empty()) {
             if (iterators_.front().next(elem)) {
@@ -136,15 +136,15 @@ class APIAsyncIterator : public APIIteratorBase<ValueType> {
         workerThread_ = std::thread(fullWorker);
     }
 
-    virtual ~APIAsyncIterator() override {
+    ~APIAsyncIterator() override {
         if (!queue_.closed()) {
             queue_.interrupt(std::make_exception_ptr(eckit::SeriousBug("Destructing incomplete async queue", Here())));
         }
         ASSERT(workerThread_.joinable());
         workerThread_.join();
     }
 
-    virtual bool next(ValueType& elem) override {
+    bool next(ValueType& elem) override {
         return !(queue_.pop(elem) == -1);
     }
 

diff --git a/src/fdb5/api/local/AxesVisitor.h b/src/fdb5/api/local/AxesVisitor.h
@@ -40,7 +40,7 @@ class AxesVisitor : public QueryVisitor<AxesElement> {
 //    bool preVisitDatabase(const eckit::URI& uri) override;
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override;
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

diff --git a/src/fdb5/api/local/ControlVisitor.h b/src/fdb5/api/local/ControlVisitor.h
@@ -39,7 +39,7 @@ class ControlVisitor : public QueryVisitor<ControlElement> {
 
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

diff --git a/src/fdb5/api/local/DumpVisitor.h b/src/fdb5/api/local/DumpVisitor.h
@@ -49,7 +49,7 @@ class DumpVisitor : public QueryVisitor<DumpElement> {
         return true;
     }
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
     void visitDatum(const Field& field, const std::string& keyFingerprint) override {
         EntryVisitor::visitDatum(field, keyFingerprint);

diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h
@@ -21,6 +21,7 @@
 
 #include "fdb5/database/DB.h"
 #include "fdb5/database/Index.h"
+#include "fdb5/database/Key.h"
 #include "fdb5/api/local/QueryVisitor.h"
 #include "fdb5/api/helpers/ListIterator.h"
 
@@ -79,12 +80,12 @@ struct ListVisitor : public QueryVisitor<ListElement> {
     }
 
     /// Test if entry matches the current request. If so, add to the output queue.
-    void visitDatum(const Field& field, const Key& key) override {
+    void visitDatum(const Field& field, const TypedKey& datumKey) override {
         ASSERT(currentCatalogue_);
         ASSERT(currentIndex_);
 
-        if (key.match(datumRequest_)) {
-            queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), key}, field.stableLocation(), field.timestamp()));
+        if (datumKey.match(datumRequest_)) {
+            queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), datumKey.canonical()}, field.stableLocation(), field.timestamp()));
         }
     }
 

diff --git a/src/fdb5/api/local/MoveVisitor.h b/src/fdb5/api/local/MoveVisitor.h
@@ -44,7 +44,7 @@ class MoveVisitor : public QueryVisitor<MoveElement> {
 
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
     void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
 
 private: // members

diff --git a/src/fdb5/api/local/PurgeVisitor.cc b/src/fdb5/api/local/PurgeVisitor.cc
@@ -82,8 +82,6 @@ void PurgeVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
     internalVisitor_->visitDatum(field, keyFingerprint);
 }
 
-void PurgeVisitor::visitDatum(const Field&, const Key&) { NOTIMP; }
-
 void PurgeVisitor::catalogueComplete(const Catalogue& catalogue) {
     internalVisitor_->catalogueComplete(catalogue);
 

diff --git a/src/fdb5/api/local/PurgeVisitor.h b/src/fdb5/api/local/PurgeVisitor.h
@@ -48,7 +48,7 @@ class PurgeVisitor : public QueryVisitor<PurgeElement> {
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field&, const Key&) override;
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

diff --git a/src/fdb5/api/local/StatsVisitor.cc b/src/fdb5/api/local/StatsVisitor.cc
@@ -47,8 +47,6 @@ void StatsVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
     internalVisitor_->visitDatum(field, keyFingerprint);
 }
 
-void StatsVisitor::visitDatum(const Field&, const Key&) { NOTIMP; }
-
 void StatsVisitor::catalogueComplete(const Catalogue& catalogue) {
     internalVisitor_->catalogueComplete(catalogue);
 

diff --git a/src/fdb5/api/local/StatsVisitor.h b/src/fdb5/api/local/StatsVisitor.h
@@ -42,7 +42,7 @@ class StatsVisitor : public QueryVisitor<StatsElement> {
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field&, const Key&) override;
+    void visitDatum(const Field&, const TypedKey&) override  { NOTIMP; }
 
 private: // members
 

diff --git a/src/fdb5/api/local/StatusVisitor.h b/src/fdb5/api/local/StatusVisitor.h
@@ -38,7 +38,7 @@ class StatusVisitor : public QueryVisitor<StatusElement> {
     bool visitEntries() override { return false; }
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; }
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
     void visitDatum(const Field& field, const std::string& keyFingerprint) override {
         EntryVisitor::visitDatum(field, keyFingerprint);