fix merge

CMakeLists.txt

@@ -20,7 +20,7 @@ endif()
 
 set( PERSISTENT_NAMESPACE "eckit" CACHE INTERNAL "" ) # needed for generating .b files for persistent support
 
-ecbuild_find_package( NAME eckit  VERSION 1.24.4 REQUIRED )
+ecbuild_find_package( NAME eckit  VERSION 1.28.3 REQUIRED )
 
 ### GRIB support
 
@@ -31,7 +31,7 @@ ecbuild_add_option( FEATURE GRIB
                     CONDITION eccodes_FOUND
                     DESCRIPTION "Support for GRIB via eccodes")
 
-ecbuild_find_package( NAME metkit VERSION 1.5 REQUIRED )
+ecbuild_find_package( NAME metkit VERSION 1.11.22 REQUIRED )
 
 ### FDB backend in CEPH object store (using Rados)
 

VERSION

@@ -1 +1 @@
-5.13.104
+5.13.105

src/fdb5/api/local/AxesVisitor.h

@@ -40,7 +40,7 @@ class AxesVisitor : public QueryVisitor<AxesElement> {
 //    bool preVisitDatabase(const eckit::URI& uri) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index&) override;
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
 
 private: // members
 

src/fdb5/api/local/ControlVisitor.h

@@ -40,7 +40,7 @@ class ControlVisitor : public QueryVisitor<ControlElement> {
     bool visitDatabase(const Catalogue& catalogue) override;
 //    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
 
 private: // members
 

src/fdb5/api/local/DumpVisitor.h

@@ -50,7 +50,7 @@ class DumpVisitor : public QueryVisitor<DumpElement> {
         return true;
     }
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
 
     void visitDatum(const Field& field, const std::string& keyFingerprint) override {
         EntryVisitor::visitDatum(field, keyFingerprint);

src/fdb5/api/local/ListVisitor.h

@@ -22,9 +22,12 @@
 #include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Key.h"
+#include "fdb5/rules/Rule.h"
 #include "fdb5/api/local/QueryVisitor.h"
 #include "fdb5/api/helpers/ListIterator.h"
 
+#include "metkit/mars/MarsRequest.h"
+
 namespace fdb5 {
 namespace api {
 namespace local {
@@ -69,25 +72,31 @@ struct ListVisitor : public QueryVisitor<ListElement> {
     bool visitIndex(const Index& index) override {
         QueryVisitor::visitIndex(index);
 
-        // Subselect the parts of the request
-        datumRequest_ = indexRequest_;
-        for (const auto& kv : index.key()) {
-            datumRequest_.unsetValues(kv.first);
-        }
 
         if (index.partialMatch(request_)) {
+
+            // Subselect the parts of the request
+            datumRequest_ = indexRequest_;
+            for (const auto& kv : index.key()) {
+                datumRequest_.unsetValues(kv.first);
+            }
+
+            // Take into account any rule-specific behaviour in the request
+            datumRequest_ = rule_->registry().canonicalise(datumRequest_);
+
             return true; // Explore contained entries
         }
+
         return false; // Skip contained entries
     }
 
     /// Test if entry matches the current request. If so, add to the output queue.
-    void visitDatum(const Field& field, const TypedKey& datumKey) override {
+    void visitDatum(const Field& field, const Key& datumKey) override {
         ASSERT(currentCatalogue_);
         ASSERT(currentIndex_);
 
         if (datumKey.match(datumRequest_)) {
-            queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), datumKey.canonical()}, field.stableLocation(), field.timestamp()));
+            queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), datumKey}, field.stableLocation(), field.timestamp()));
         }
     }
 

src/fdb5/api/local/MoveVisitor.h

@@ -45,7 +45,7 @@ class MoveVisitor : public QueryVisitor<MoveElement> {
 //    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
     void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
 
 private: // members

src/fdb5/api/local/PurgeVisitor.h

@@ -48,7 +48,7 @@ class PurgeVisitor : public QueryVisitor<PurgeElement> {
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
 
 private: // members
 

src/fdb5/api/local/StatsVisitor.h

@@ -43,7 +43,7 @@ class StatsVisitor : public QueryVisitor<StatsElement> {
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field&, const TypedKey&) override  { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override  { NOTIMP; }
 
 private: // members
 

src/fdb5/api/local/StatusVisitor.h

@@ -39,7 +39,7 @@ class StatusVisitor : public QueryVisitor<StatusElement> {
     bool visitDatabase(const Catalogue& catalogue) override  { queue_.emplace(catalogue); return true; }
 //    bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; }
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
 
     void visitDatum(const Field& field, const std::string& keyFingerprint) override {
         EntryVisitor::visitDatum(field, keyFingerprint);

src/fdb5/api/local/WipeVisitor.h

@@ -51,7 +51,7 @@ class WipeVisitor : public QueryVisitor<WipeElement> {
     bool visitDatabase(const Catalogue& catalogue) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
     void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
 
     void onDatabaseNotFound(const fdb5::DatabaseNotFoundException& e) override { throw e; }

src/fdb5/daos/DaosCatalogue.cc

@@ -167,7 +167,7 @@ std::vector<Index> DaosCatalogue::indexes(bool) const {
 
 std::string DaosCatalogue::type() const {
 
-    return DaosCatalogue::catalogueTypeName();
+    // return DaosCatalogue::catalogueTypeName();
 
 }
 

src/fdb5/daos/DaosCatalogue.h

@@ -32,7 +32,7 @@ class DaosCatalogue : public CatalogueImpl, public DaosCommon {
     DaosCatalogue(const Key& key, const fdb5::Config& config);
     DaosCatalogue(const eckit::URI& uri, const ControlIdentifiers& controlIdentifiers, const fdb5::Config& config);
 
-    static const char* catalogueTypeName() { return fdb5::DaosEngine::typeName(); }
+    // static const char* catalogueTypeName() { return fdb5::DaosEngine::typeName(); }
 
     eckit::URI uri() const override;
     const Key& indexKey() const override { return currentIndexKey_; }

src/fdb5/daos/DaosIndex.h

@@ -41,8 +41,8 @@ class DaosIndex : public IndexBase {
 
     bool dirty() const override { NOTIMP; }
 
-    void open() override { NOTIMP; };
-    void close() override { NOTIMP; }
+    void open() override { NOTIMP; }
+    void close() override {}
     void reopen() override { NOTIMP; }
 
     void visit(IndexLocationVisitor& visitor) const override { NOTIMP; }

src/fdb5/database/Catalogue.cc

@@ -13,6 +13,7 @@
 
 #include "eckit/config/Resource.h"
 #include "eckit/exception/Exceptions.h"
+#include "eckit/io/AutoCloser.h"
 #include "eckit/thread/AutoLock.h"
 #include "eckit/thread/Mutex.h"
 #include "eckit/utils/StringTools.h"
@@ -37,11 +38,18 @@ void Catalogue::visitEntries(EntryVisitor& visitor /*, const Store& store*/, boo
 
     std::vector<Index> all = indexes(sorted);
 
+    // It is likely that many indexes in the same database share resources/files/etc.
+    // To prevent repeated opening/closing (especially where a PooledFile would facilitate things)
+    // pre-open the indexes, and keep them open
+    std::vector<eckit::AutoCloser<Index>> closers;
+    closers.reserve(all.size());
+
     // Allow the visitor to selectively reject this DB.
     if (visitor.visitDatabase(*this)) {
         if (visitor.visitIndexes()) {
-            for (const Index& idx : all) {
+            for (Index& idx : all) {
                 if (visitor.visitEntries()) {
+                    closers.emplace_back(idx);
                     idx.entries(visitor); // contains visitIndex
                 } else {
                     visitor.visitIndex(idx);

src/fdb5/database/EntryVisitMechanism.cc

@@ -52,6 +52,8 @@ Store& EntryVisitor::store() const {
 bool EntryVisitor::visitDatabase(const Catalogue& catalogue) {
     currentCatalogue_ = &catalogue;
     currentStore_ = nullptr;
+    currentIndex_ = nullptr;
+    rule_ = nullptr;
     return true;
 }
 
@@ -63,19 +65,20 @@ void EntryVisitor::catalogueComplete(const Catalogue& catalogue) {
     delete currentStore_;
     currentStore_ = nullptr;
     currentIndex_ = nullptr;
+    rule_ = nullptr;
 }
 
 bool EntryVisitor::visitIndex(const Index& index) {
     currentIndex_ = &index;
+    rule_ = currentCatalogue_->schema().ruleFor(currentCatalogue_->key(), currentIndex_->key());
     return true;
 }
 
 void EntryVisitor::visitDatum(const Field& field, const std::string& keyFingerprint) {
     ASSERT(currentCatalogue_);
     ASSERT(currentIndex_);
-    const Rule* rule = currentCatalogue_->schema().ruleFor(currentCatalogue_->key(), currentIndex_->key());
-    ASSERT(rule);
-    TypedKey key(keyFingerprint, *rule);
+    ASSERT(rule_);
+    Key key(keyFingerprint, *rule_);
     visitDatum(field, key);
 }
 

src/fdb5/database/EntryVisitMechanism.h

@@ -56,15 +56,15 @@ class EntryVisitor : public eckit::NonCopyable {
 
 private: // methods
 
-    virtual void visitDatum(const Field& field, const TypedKey& datumKey) = 0;
+    virtual void visitDatum(const Field& field, const Key& datumKey) = 0;
 
 protected:  // members
 
     // n.b. non-owning
     const Catalogue* currentCatalogue_ = nullptr;
     mutable Store* currentStore_ = nullptr;
     const Index* currentIndex_ = nullptr;
-
+    const Rule* rule_ = nullptr;
 };
 
 //----------------------------------------------------------------------------------------------------------------------

src/fdb5/database/Key.cc

@@ -28,6 +28,14 @@ namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
 
+BaseKey::BaseKey(const std::string& fingerprint, const Rule& rule) {
+    eckit::Tokenizer parse(":", true);
+    eckit::StringList values;
+    parse(fingerprint, values);
+
+    rule.fill(*this, values);
+}
+
 void BaseKey::decode(eckit::Stream& s) {
 
     keys_.clear();
@@ -45,6 +53,7 @@ void BaseKey::decode(eckit::Stream& s) {
     }
 
     s >> n;
+    names_.reserve(n);
     for (size_t i = 0; i < n; ++i) {
         s >> k;
         s >> v; // this is the type (ignoring FTM)
@@ -148,7 +157,7 @@ bool BaseKey::match(const metkit::mars::MarsRequest& request) const {
         }
 
         bool found=false;
-        auto values = request.values(k);
+        const auto& values = request.values(k);
         std::string can = canonicalise(k, j->second);
         for (auto it = values.cbegin(); !found && it != values.cend(); it++) {
             found = can == canonicalise(k, *it);
@@ -307,6 +316,9 @@ Key::Key(eckit::Stream& s) {
 Key::Key(std::initializer_list<std::pair<const std::string, std::string>> l) :
     BaseKey(l) {}
 
+Key::Key(const std::string& fingerprint, const Rule& rule) :
+    BaseKey(fingerprint, rule) {}
+
 Key Key::parseString(const std::string& s) {
 
     eckit::Tokenizer parse1(",");
@@ -342,13 +354,8 @@ TypedKey::TypedKey(const TypesRegistry& reg) :
     registry_(std::cref(reg)) {}
 
 TypedKey::TypedKey(const std::string &s, const Rule& rule) :
+    BaseKey(s, rule),
     registry_(std::cref(rule.registry())) {
-
-    eckit::Tokenizer parse(":", true);
-    eckit::StringList values;
-    parse(s, values);
-
-    rule.fill(*this, values);
 }
 
 TypedKey::TypedKey(const eckit::StringDict &keys, const TypesRegistry& reg) :

src/fdb5/database/Key.h

@@ -61,6 +61,7 @@ class BaseKey {
             names_.emplace_back(k.first);
         }
     }
+    BaseKey(const std::string& fingerprint, const Rule& rule);
 
     virtual ~BaseKey() = default;
 
@@ -167,6 +168,7 @@ class Key : public BaseKey {
     explicit Key() = default;
     explicit Key(eckit::Stream &);
     explicit Key(const eckit::StringDict &keys);
+    explicit Key(const std::string& fingerprint, const Rule& rule);
     Key(std::initializer_list<std::pair<const std::string, std::string>>);
 
     static Key parseString(const std::string& s);

src/fdb5/database/MoveVisitor.h

@@ -38,7 +38,7 @@ class MoveVisitor : public EntryVisitor {
     bool visitEntries() override { return false; }
 
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
     void visitDatum(const Field& /*field*/, const std::string& /*keyFingerprint*/) override { NOTIMP; }
 
 protected: // members

src/fdb5/database/WipeVisitor.h

@@ -37,7 +37,7 @@ class WipeVisitor : public EntryVisitor {
     ~WipeVisitor() override;
 
     bool visitEntries() override { return false; }
-    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
     void visitDatum(const Field& /*field*/, const std::string& /*keyFingerprint*/) override { NOTIMP; }
 
 protected: // members

src/fdb5/rules/MatchOptional.cc

@@ -65,7 +65,7 @@ bool MatchOptional::optional() const {
     return true;
 }
 
-void MatchOptional::fill(TypedKey& key, const std::string &keyword, const std::string& value) const {
+void MatchOptional::fill(BaseKey& key, const std::string &keyword, const std::string& value) const {
     if (!value.empty()) {
         key.push(keyword, value);
     }

src/fdb5/rules/MatchOptional.h

@@ -50,7 +50,7 @@ class MatchOptional : public Matcher{
     const std::vector<std::string>& values(const metkit::mars::MarsRequest& rq, const std::string& keyword) const override;
     void print( std::ostream& out ) const override;
     const std::string& defaultValue() const override;
-    void fill(TypedKey& key, const std::string& keyword, const std::string& value) const override;
+    void fill(BaseKey& key, const std::string& keyword, const std::string& value) const override;
 
 private: // members
 

src/fdb5/rules/Matcher.cc

@@ -48,7 +48,7 @@ const std::vector<std::string> &Matcher::values(const metkit::mars::MarsRequest&
     return rq.values(keyword);
 }
 
-void Matcher::fill(TypedKey& key, const std::string &keyword, const std::string& value) const {
+void Matcher::fill(BaseKey& key, const std::string &keyword, const std::string& value) const {
     key.push(keyword, value);
 }
 

src/fdb5/rules/Matcher.h

@@ -31,8 +31,7 @@ namespace mars {
 
 namespace fdb5 {
 
-class Key;
-class TypedKey;
+class BaseKey;
 class Key;
 class TypesRegistry;
 
@@ -54,7 +53,7 @@ class Matcher : public eckit::Streamable {
     virtual const std::string &defaultValue() const;
 
     virtual bool match(const std::string &keyword, const Key& key) const = 0;
-    virtual void fill(TypedKey& key, const std::string &keyword, const std::string& value) const;
+    virtual void fill(BaseKey& key, const std::string &keyword, const std::string& value) const;
 
 
     virtual void dump(std::ostream &s, const std::string &keyword, const TypesRegistry &registry) const = 0;

src/fdb5/rules/Predicate.cc

@@ -68,7 +68,7 @@ const std::vector<std::string>& Predicate::values(const metkit::mars::MarsReques
     return matcher_->values(rq, keyword_);
 }
 
-void Predicate::fill(TypedKey& key, const std::string& value) const {
+void Predicate::fill(BaseKey& key, const std::string& value) const {
     matcher_->fill(key, keyword_, value);
 }