Merge branch 'release/5.12.0' into develop

VERSION

@@ -1 +1 @@
-5.11.30
+5.12.0

src/fdb5/CMakeLists.txt

@@ -32,6 +32,8 @@ list( APPEND fdb5_srcs
     api/SelectFDB.h
 
     api/helpers/APIIterator.h
+    api/helpers/Axesiterator.cc
+    api/helpers/AxesIterator.h
     api/helpers/ControlIterator.cc
     api/helpers/ControlIterator.h
     api/helpers/FDBToolRequest.cc
@@ -50,6 +52,8 @@ list( APPEND fdb5_srcs
     api/local/QueryVisitor.h
     api/local/QueueStringLogTarget.h
     api/local/ListVisitor.h
+    api/local/AxesVisitor.cc
+    api/local/AxesVisitor.h
     api/local/ControlVisitor.cc
     api/local/ControlVisitor.h
     api/local/DumpVisitor.h
@@ -408,6 +412,7 @@ if(HAVE_FDB_BUILD_TOOLS)
 endif()
 
 list( APPEND fdb5_tools
+    fdb-axes
     fdb-write
     fdb-copy
     fdb-dump

src/fdb5/api/FDB.cc

@@ -295,6 +295,16 @@ void FDB::flush() {
     }
 }
 
+IndexAxis FDB::axes(const FDBToolRequest& request, int level) {
+    IndexAxis axes;
+    AxesElement elem;
+    auto it = internal_->axes(request, level);
+    while (it.next(elem)) {
+        axes.merge(elem.axes());
+    }
+    return axes;
+}
+
 bool FDB::dirty() const {
     return dirty_;
 }

src/fdb5/api/FDB.h

@@ -109,6 +109,9 @@ class FDB {
     ControlIterator control(const FDBToolRequest& request,
                             ControlAction action,
                             ControlIdentifiers identifiers);
+
+    IndexAxis axes(const FDBToolRequest& request, int level=3);
+
     bool enabled(const ControlIdentifier& controlIdentifier) const;
 
     bool dirty() const;

src/fdb5/api/FDBFactory.h

@@ -28,7 +28,7 @@
 #include "fdb5/database/DB.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/api/FDBStats.h"
-#include "fdb5/api/helpers/ListIterator.h"
+#include "fdb5/api/helpers/AxesIterator.h"
 #include "fdb5/api/helpers/ListIterator.h"
 #include "fdb5/api/helpers/ControlIterator.h"
 #include "fdb5/api/helpers/DumpIterator.h"
@@ -88,6 +88,8 @@ class FDBBase : private eckit::NonCopyable {
 
     virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0;
 
+    virtual AxesIterator axes(const FDBToolRequest& request, int axes) { NOTIMP; }
+
     // -------------- API management ----------------------------
 
     /// ID used for hashing in the Rendezvous hash. Should be unique amongst those used

src/fdb5/api/LocalFDB.cc

@@ -29,6 +29,7 @@
 #include "fdb5/rules/Schema.h"
 #include "fdb5/LibFdb5.h"
 
+#include "fdb5/api/local/AxesVisitor.h"
 #include "fdb5/api/local/ControlVisitor.h"
 #include "fdb5/api/local/DumpVisitor.h"
 #include "fdb5/api/local/ListVisitor.h"
@@ -123,6 +124,11 @@ ControlIterator LocalFDB::control(const FDBToolRequest& request,
     return queryInternal<ControlVisitor>(request, action, identifiers);
 }
 
+AxesIterator LocalFDB::axes(const FDBToolRequest& request, int level) {
+    LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axes() : " << request << std::endl;
+    return queryInternal<AxesVisitor>(request, config_, level);
+}
+
 void LocalFDB::flush() {
     if (archiver_) {
         archiver_->flush();

src/fdb5/api/LocalFDB.h

@@ -59,6 +59,8 @@ class LocalFDB : public FDBBase {
 
     MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override;
 
+    AxesIterator axes(const FDBToolRequest& request, int axes) override;
+
     void flush() override;
 
 private: // methods

src/fdb5/api/helpers/AxesIterator.cc

@@ -0,0 +1,41 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the EC H2020 funded project NextGenIO
+ * (Project ID: 671951) www.nextgenio.eu
+ */
+
+#include "fdb5/api/helpers/AxesIterator.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+AxesElement::AxesElement(Key&& dbKey, IndexAxis&& axes) :
+        dbKey_(std::move(dbKey)), axes_(std::move(axes)) {}
+
+AxesElement::AxesElement(eckit::Stream& s) {
+    s >> dbKey_;
+    axes_ = IndexAxis(s, IndexAxis::currentVersion());
+}
+
+void AxesElement::print(std::ostream& out) const {
+    out << "Axes(db=" << dbKey_ << ", axes=" << axes_ << ")";
+}
+
+void AxesElement::encode(eckit::Stream& s) const {
+    s << dbKey_;
+    axes_.encode(s, IndexAxis::currentVersion());
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5

src/fdb5/api/helpers/AxesIterator.h

@@ -0,0 +1,74 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Simon Smart
+/// @date   January 2024
+
+#pragma once
+
+#include "fdb5/database/Key.h"
+#include "fdb5/database/IndexAxis.h"
+#include "fdb5/api/helpers/APIIterator.h"
+
+namespace eckit {
+class Stream;
+class JSON;
+}
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class AxesElement {
+public: // methods
+
+    AxesElement() = default;
+    AxesElement(Key&& dbKey, IndexAxis&& axis);
+    explicit AxesElement(eckit::Stream& s);
+
+    [[ nodiscard ]]
+    const Key& key() const { return dbKey_; }
+
+    [[ nodiscard ]]
+    const IndexAxis& axes() const { return axes_; }
+
+    void print(std::ostream& out) const;
+
+private: // methods
+
+    void encode(eckit::Stream& s) const;
+
+    friend std::ostream& operator<<(std::ostream& os, const AxesElement& e) {
+        e.print(os);
+        return os;
+    }
+
+    friend eckit::Stream& operator<<(eckit::Stream& s, const AxesElement& r) {
+        r.encode(s);
+        return s;
+    }
+
+private: // members
+
+    Key dbKey_;
+    IndexAxis axes_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+using AxesAggregateIterator = APIAggregateIterator<AxesElement>;
+
+using AxesAsyncIterator = APIAsyncIterator<AxesElement>;
+
+using AxesIterator = APIIterator<AxesElement>;
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace fdb5

src/fdb5/api/local/AxesVisitor.cc

@@ -0,0 +1,82 @@
+/*
+ * (C) Copyright 2018- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include "fdb5/api/local/AxesVisitor.h"
+
+#include "fdb5/database/Catalogue.h"
+
+namespace fdb5 {
+namespace api {
+namespace local {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+AxesVisitor::AxesVisitor(eckit::Queue<AxesElement>& queue,
+                         const metkit::mars::MarsRequest& request,
+                         const Config& config,
+                         int level) :
+        QueryVisitor<AxesElement>(queue, request),
+        schema_(config.schema()),
+        level_(level) {}
+
+#if 0
+
+// TODO: Here we can do nice tricks to make things go muuuuuuuuuuuuuch faster...
+//       See improvements to the EntryVisitMechanism... & the schema
+
+bool AxesVisitor::preVisitDatabase(const eckit::URI& uri) {
+
+    // If level == 1, avoid constructing the Catalogue/Store objects, so just interrogate the URIs
+    if (level_ == 1 && uri.scheme() == "toc") {
+        // TODO: This is hacky, only works with the toc backend...
+        if (schema_.matchFirstLevel(uri.path().baseName(), dbKey_)) {
+            axes_.wipe();
+            axes_.insert(dbKey_);
+            axes_.sort();
+            queue_.emplace(AxesElement{std::move(dbKey_), std::move(axes_)});
+        }
+        return false;
+    }
+
+    return true;
+}
+#endif
+
+bool AxesVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) {
+    dbKey_ = catalogue.key();
+    axes_.wipe();
+    axes_.insert(dbKey_);
+    axes_.sort();
+    return (level_ > 1);
+}
+
+bool AxesVisitor::visitIndex(const Index& index) {
+    if (index.partialMatch(request_)) {
+        IndexAxis tmpAxis;
+        tmpAxis.insert(index.key());
+        tmpAxis.sort();
+        axes_.merge(tmpAxis);   // avoid sorts on the (growing) main Axes object
+
+        if (level_ > 2) {
+            axes_.merge(index.axes());
+        }
+    }
+    return false;
+}
+
+void AxesVisitor::catalogueComplete(const fdb5::Catalogue& catalogue) {
+    queue_.emplace(AxesElement{std::move(dbKey_), std::move(axes_)});
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace local
+} // namespace api
+} // namespace fdb5

src/fdb5/api/local/AxesVisitor.h

@@ -0,0 +1,57 @@
+/*
+ * (C) Copyright 2018- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/// @author Simon Smart
+/// @date   January 2024
+
+#pragma once
+
+#include "fdb5/api/local/QueryVisitor.h"
+#include "fdb5/api/helpers/AxesIterator.h"
+
+
+namespace fdb5 {
+namespace api {
+namespace local {
+
+/// @note Helper classes for LocalFDB
+
+//----------------------------------------------------------------------------------------------------------------------
+
+class AxesVisitor : public QueryVisitor<AxesElement> {
+public:
+
+    AxesVisitor(eckit::Queue<AxesElement>& queue,
+                const metkit::mars::MarsRequest& request,
+                const Config& config,
+                int level);
+
+    bool visitIndexes() override { return true; }
+    bool visitEntries() override { return false; }
+    void catalogueComplete(const fdb5::Catalogue& catalogue) override;
+
+//    bool preVisitDatabase(const eckit::URI& uri) override;
+    bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
+    bool visitIndex(const Index&) override;
+    void visitDatum(const Field&, const Key&) override { NOTIMP; }
+
+private: // members
+
+    Key dbKey_;
+    IndexAxis axes_;
+    const Schema& schema_;
+    int level_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+} // namespace local
+} // namespace api
+} // namespace fdb5