Merge pull request #26 from ecmwf/feature/canonicalKey

FDB-303 added decoupling between Key and fully typed 'CanonicalKey' improved TypesRegistry lifecycle management
ecmwf · Sep 23, 2024 · a310849 · a310849
2 parents 7f8e64c + ddc823a
commit a310849
Show file tree

Hide file tree

Showing 197 changed files with 2,016 additions and 1,340 deletions.
diff --git a/src/fdb5/LibFdb5.h b/src/fdb5/LibFdb5.h
@@ -20,7 +20,6 @@
 #include "eckit/system/Library.h"
 
 #include "fdb5/database/DB.h"
-#include "fdb5/types/TypesRegistry.h"
 #include "fdb5/api/helpers/Callback.h"
 
 namespace fdb5 {

diff --git a/src/fdb5/api/DistFDB.h b/src/fdb5/api/DistFDB.h
@@ -69,7 +69,7 @@ class DistFDB : public FDBBase {
 
 private: // methods
 
-    virtual void print(std::ostream& s) const override;
+    void print(std::ostream& s) const override;
 
     template <typename QueryFN>
     auto queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> decltype(fn(*(FDB*)(nullptr), request));

diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc
@@ -108,15 +108,15 @@ void FDB::archive(const Key& key, const void* data, size_t length) {
     // This is the API entrypoint. Keys supplied by the user may not have type registry info attached (so
     // serialisation won't work properly...)
     Key keyInternal(key);
-    keyInternal.registry(config().schema().registry());
 
     // step in archival requests from the model is just an integer. We need to include the stepunit
     auto stepunit = keyInternal.find("stepunits");
     if (stepunit != keyInternal.end()) {
-        if (stepunit->second.size()>0 && stepunit->second[0]!='h') {
+        if (stepunit->second.size()>0 && static_cast<char>(tolower(stepunit->second[0])) != 'h') {
             auto step = keyInternal.find("step");
             if (step != keyInternal.end()) {
-                std::string canonicalStep = keyInternal.registry().lookupType("step").toKey("step", step->second+stepunit->second);
+                std::string canonicalStep = config().schema().registry().lookupType("step").toKey(step->second + static_cast<char>(tolower(stepunit->second[0])));
+                keyInternal.set("step", canonicalStep);
             }
         }
         keyInternal.unset("stepunits");

diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h
@@ -185,7 +185,7 @@ class FDBBuilder : public FDBBuilderBase {
 
 private: // methods
 
-    virtual std::unique_ptr<FDBBase> make(const Config& config) const override {
+    std::unique_ptr<FDBBase> make(const Config& config) const override {
         return std::unique_ptr<T>(new T(config, name_));
     }
 };

diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc
@@ -968,7 +968,7 @@ class FDBRemoteDataHandle : public DataHandle {
         overallPosition_(0),
         currentBuffer_(0),
         complete_(false) {}
-    virtual bool canSeek() const override { return false; }
+    bool canSeek() const override { return false; }
 
 private: // methods
 

diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h
@@ -126,9 +126,9 @@ class RemoteFDB : public FDBBase {
     void sendArchiveData(uint32_t id, const Key& key, const void* data, size_t length);
     long sendArchiveData(uint32_t id, const std::vector<std::pair<Key, eckit::Buffer>>& elements, size_t count);
 
-    virtual void print(std::ostream& s) const override;
+    void print(std::ostream& s) const override;
 
-    virtual FDBStats stats() const override;
+    FDBStats stats() const override;
 
 private: // members
 

diff --git a/src/fdb5/api/SelectFDB.cc b/src/fdb5/api/SelectFDB.cc
@@ -208,7 +208,7 @@ void SelectFDB::print(std::ostream &s) const {
     s << "SelectFDB()";
 }
 
-bool SelectFDB::matches(const Key &key, const SelectMap &select, bool requireMissing) const {
+bool SelectFDB::matches(const Key& key, const SelectMap &select, bool requireMissing) const {
 
     for (const auto& kv : select) {
 

diff --git a/src/fdb5/api/helpers/APIIterator.h b/src/fdb5/api/helpers/APIIterator.h
@@ -83,9 +83,9 @@ class APIAggregateIterator : public APIIteratorBase<ValueType> {
     APIAggregateIterator(std::queue<APIIterator<ValueType>>&& iterators) :
         iterators_(std::move(iterators)) {}
 
-    virtual ~APIAggregateIterator() override {}
+    ~APIAggregateIterator() override {}
 
-    virtual bool next(ValueType& elem) override {
+    bool next(ValueType& elem) override {
 
         while (!iterators_.empty()) {
             if (iterators_.front().next(elem)) {
@@ -136,15 +136,15 @@ class APIAsyncIterator : public APIIteratorBase<ValueType> {
         workerThread_ = std::thread(fullWorker);
     }
 
-    virtual ~APIAsyncIterator() override {
+    ~APIAsyncIterator() override {
         if (!queue_.closed()) {
             queue_.interrupt(std::make_exception_ptr(eckit::SeriousBug("Destructing incomplete async queue", Here())));
         }
         ASSERT(workerThread_.joinable());
         workerThread_.join();
     }
 
-    virtual bool next(ValueType& elem) override {
+    bool next(ValueType& elem) override {
         return !(queue_.pop(elem) == -1);
     }
 

diff --git a/src/fdb5/api/local/AxesVisitor.h b/src/fdb5/api/local/AxesVisitor.h
@@ -40,7 +40,7 @@ class AxesVisitor : public QueryVisitor<AxesElement> {
 //    bool preVisitDatabase(const eckit::URI& uri) override;
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override;
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

diff --git a/src/fdb5/api/local/ControlVisitor.h b/src/fdb5/api/local/ControlVisitor.h
@@ -39,7 +39,7 @@ class ControlVisitor : public QueryVisitor<ControlElement> {
 
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

diff --git a/src/fdb5/api/local/DumpVisitor.h b/src/fdb5/api/local/DumpVisitor.h
@@ -49,7 +49,7 @@ class DumpVisitor : public QueryVisitor<DumpElement> {
         return true;
     }
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
     void visitDatum(const Field& field, const std::string& keyFingerprint) override {
         EntryVisitor::visitDatum(field, keyFingerprint);

diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h
@@ -21,6 +21,7 @@
 
 #include "fdb5/database/DB.h"
 #include "fdb5/database/Index.h"
+#include "fdb5/database/Key.h"
 #include "fdb5/api/local/QueryVisitor.h"
 #include "fdb5/api/helpers/ListIterator.h"
 
@@ -79,12 +80,12 @@ struct ListVisitor : public QueryVisitor<ListElement> {
     }
 
     /// Test if entry matches the current request. If so, add to the output queue.
-    void visitDatum(const Field& field, const Key& key) override {
+    void visitDatum(const Field& field, const TypedKey& datumKey) override {
         ASSERT(currentCatalogue_);
         ASSERT(currentIndex_);
 
-        if (key.match(datumRequest_)) {
-            queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), key}, field.stableLocation(), field.timestamp()));
+        if (datumKey.match(datumRequest_)) {
+            queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), datumKey.canonical()}, field.stableLocation(), field.timestamp()));
         }
     }
 

diff --git a/src/fdb5/api/local/MoveVisitor.h b/src/fdb5/api/local/MoveVisitor.h
@@ -44,7 +44,7 @@ class MoveVisitor : public QueryVisitor<MoveElement> {
 
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
     void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
 
 private: // members

diff --git a/src/fdb5/api/local/PurgeVisitor.cc b/src/fdb5/api/local/PurgeVisitor.cc
@@ -82,8 +82,6 @@ void PurgeVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
     internalVisitor_->visitDatum(field, keyFingerprint);
 }
 
-void PurgeVisitor::visitDatum(const Field&, const Key&) { NOTIMP; }
-
 void PurgeVisitor::catalogueComplete(const Catalogue& catalogue) {
     internalVisitor_->catalogueComplete(catalogue);
 

diff --git a/src/fdb5/api/local/PurgeVisitor.h b/src/fdb5/api/local/PurgeVisitor.h
@@ -48,7 +48,7 @@ class PurgeVisitor : public QueryVisitor<PurgeElement> {
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field&, const Key&) override;
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

diff --git a/src/fdb5/api/local/StatsVisitor.cc b/src/fdb5/api/local/StatsVisitor.cc
@@ -47,8 +47,6 @@ void StatsVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
     internalVisitor_->visitDatum(field, keyFingerprint);
 }
 
-void StatsVisitor::visitDatum(const Field&, const Key&) { NOTIMP; }
-
 void StatsVisitor::catalogueComplete(const Catalogue& catalogue) {
     internalVisitor_->catalogueComplete(catalogue);
 

diff --git a/src/fdb5/api/local/StatsVisitor.h b/src/fdb5/api/local/StatsVisitor.h
@@ -42,7 +42,7 @@ class StatsVisitor : public QueryVisitor<StatsElement> {
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field&, const Key&) override;
+    void visitDatum(const Field&, const TypedKey&) override  { NOTIMP; }
 
 private: // members
 

diff --git a/src/fdb5/api/local/StatusVisitor.h b/src/fdb5/api/local/StatusVisitor.h
@@ -38,7 +38,7 @@ class StatusVisitor : public QueryVisitor<StatusElement> {
     bool visitEntries() override { return false; }
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; }
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
     void visitDatum(const Field& field, const std::string& keyFingerprint) override {
         EntryVisitor::visitDatum(field, keyFingerprint);

diff --git a/src/fdb5/api/local/WipeVisitor.h b/src/fdb5/api/local/WipeVisitor.h
@@ -50,10 +50,10 @@ class WipeVisitor : public QueryVisitor<WipeElement> {
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
-    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
     void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
 
-    virtual void onDatabaseNotFound(const fdb5::DatabaseNotFoundException& e) override { throw e; }
+    void onDatabaseNotFound(const fdb5::DatabaseNotFoundException& e) override { throw e; }
 
 private: // members
 

diff --git a/src/fdb5/config/Config.h b/src/fdb5/config/Config.h
@@ -41,7 +41,7 @@ class Config : public eckit::LocalConfiguration {
     /// may involve loading a specific config.json
     Config expandConfig() const;
 
-    virtual ~Config() override;
+    ~Config() override;
 
     /// Given paths of the form ~fdb, if FDB_HOME has been expanded in the configuration
     /// then do the expansion in here.

diff --git a/src/fdb5/daos/DaosArrayHandle.h b/src/fdb5/daos/DaosArrayHandle.h
@@ -34,32 +34,32 @@ class DaosArrayHandle : public eckit::DataHandle {
 
     ~DaosArrayHandle();
 
-    virtual void print(std::ostream&) const override;
+    void print(std::ostream&) const override;
 
-    virtual void openForWrite(const eckit::Length&) override;
-    virtual void openForAppend(const eckit::Length&) override { NOTIMP; };
-    virtual eckit::Length openForRead() override;
+    void openForWrite(const eckit::Length&) override;
+    void openForAppend(const eckit::Length&) override { NOTIMP; };
+    eckit::Length openForRead() override;
 
-    virtual long write(const void*, long) override;
-    virtual long read(void*, long) override;
-    virtual void close() override;
-    virtual void flush() override;
+    long write(const void*, long) override;
+    long read(void*, long) override;
+    void close() override;
+    void flush() override;
 
-    virtual eckit::Length size() override;
-    virtual eckit::Length estimate() override;
-    virtual eckit::Offset position() override;
-    virtual eckit::Offset seek(const eckit::Offset&) override;
-    virtual bool canSeek() const override;
-    // virtual void skip(const eckit::Length&) override;
+    eckit::Length size() override;
+    eckit::Length estimate() override;
+    eckit::Offset position() override;
+    eckit::Offset seek(const eckit::Offset&) override;
+    bool canSeek() const override;
+    // void skip(const eckit::Length&) override;
 
-    // virtual void rewind() override;
-    // virtual void restartReadFrom(const Offset&) override;
-    // virtual void restartWriteFrom(const Offset&) override;
+    // void rewind() override;
+    // void restartReadFrom(const Offset&) override;
+    // void restartWriteFrom(const Offset&) override;
 
-    virtual std::string title() const override;
+    std::string title() const override;
 
-    // virtual void encode(Stream&) const override;
-    // virtual const ReanimatorBase& reanimator() const override { return reanimator_; }
+    // void encode(Stream&) const override;
+    // const ReanimatorBase& reanimator() const override { return reanimator_; }
 
     // static const ClassSpec& classSpec() { return classSpec_; }
 

diff --git a/src/fdb5/daos/DaosArrayPartHandle.h b/src/fdb5/daos/DaosArrayPartHandle.h
@@ -39,32 +39,32 @@ class DaosArrayPartHandle : public eckit::DataHandle {
 
     ~DaosArrayPartHandle();
 
-    virtual void print(std::ostream&) const override;
+    void print(std::ostream&) const override;
 
-    // virtual void openForWrite(const eckit::Length&) override;
-    // virtual void openForAppend(const eckit::Length&) override;
-    virtual eckit::Length openForRead() override;
+    // void openForWrite(const eckit::Length&) override;
+    // void openForAppend(const eckit::Length&) override;
+    eckit::Length openForRead() override;
 
-    // virtual long write(const void*, long) override;
-    virtual long read(void*, long) override;
-    virtual void close() override;
-    virtual void flush() override;
+    // long write(const void*, long) override;
+    long read(void*, long) override;
+    void close() override;
+    void flush() override;
 
-    virtual eckit::Length size() override;
-    virtual eckit::Length estimate() override;
-    virtual eckit::Offset position() override;
-    virtual eckit::Offset seek(const eckit::Offset&) override;
-    virtual bool canSeek() const override;
-    // virtual void skip(const eckit::Length&) override;
+    eckit::Length size() override;
+    eckit::Length estimate() override;
+    eckit::Offset position() override;
+    eckit::Offset seek(const eckit::Offset&) override;
+    bool canSeek() const override;
+    // void skip(const eckit::Length&) override;
 
-    // virtual void rewind() override;
-    // virtual void restartReadFrom(const Offset&) override;
-    // virtual void restartWriteFrom(const Offset&) override;
+    // void rewind() override;
+    // void restartReadFrom(const Offset&) override;
+    // void restartWriteFrom(const Offset&) override;
 
-    virtual std::string title() const override;
+    std::string title() const override;
 
-    // virtual void encode(Stream&) const override;
-    // virtual const ReanimatorBase& reanimator() const override { return reanimator_; }
+    // void encode(Stream&) const override;
+    // const ReanimatorBase& reanimator() const override { return reanimator_; }
 
     // static const ClassSpec& classSpec() { return classSpec_; }
 

diff --git a/src/fdb5/daos/DaosCatalogue.cc b/src/fdb5/daos/DaosCatalogue.cc
@@ -155,7 +155,7 @@ std::vector<Index> DaosCatalogue::indexes(bool) const {
             /// @todo: the index_kv may exist even if it does not have the "key" key
         }
 
-        res.push_back(Index(new fdb5::DaosIndex(index_key.value(), index_kv_name, false)));
+        res.push_back(Index(new fdb5::DaosIndex(index_key.value(), *this, index_kv_name, false)));
 
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -155,7 +155,7 @@ std::vector<Index> DaosCatalogue::indexes(bool) const { @@
                 /// @todo: the index_kv may exist even if it does not have the "key" key
             }
-            res.push_back(Index(new fdb5::DaosIndex(index_key.value(), index_kv_name, false)));
+            res.push_back(Index(new fdb5::DaosIndex(index_key.value(), *this, index_kv_name, false)));
         }
@@ Expand Down @@