feat(FAM): WIP catalogue

fam_test_dir/config.yaml

@@ -0,0 +1,18 @@
+---
+type: local
+schema: /home/macm/workspace/opencube/dev/bundle_fdb_openfam/fdb/fam_test_dir/schema
+store: fam
+catalogue: toc
+spaces:
+- handler: Default
+  roots:
+  - path: ./root
+  - path: fam://endpoint/region
+  - path: ./backup
+    wipe: true
+    list: true
+    retrieve: false
+fam_roots:
+- uri: fam://127.0.0.1:8880/test_region_fdb
+  writable: true
+  visit: true

fam_test_dir/schema

@@ -0,0 +1,3 @@
+[ fam1a, fam1b, fam1c
+    [ fam2a, fam2b, fam2c
+       [ fam3a, fam3b, fam3c ]]]

src/fdb5/fam/CMakeLists.txt

@@ -1,11 +1,25 @@
 if( HAVE_FAMFDB )
     list( APPEND fdb5_srcs
-        fam/FamCommon.h
+        fam/FamCatalogue.h
+        fam/FamCatalogue.cc
         fam/FamCommon.cc
-        fam/FamStore.h
-        fam/FamStore.cc
-        fam/FamFieldLocation.h
+        fam/FamCommon.h
+        fam/FamEngine.cc
+        fam/FamEngine.h
         fam/FamFieldLocation.cc
+        fam/FamFieldLocation.h
+        fam/FamIndex.cc
+        fam/FamIndex.h
+        fam/FamIndexLocation.cc
+        fam/FamIndexLocation.h
+        fam/FamRecord.cc
+        fam/FamRecord.h
+        fam/FamRecordVersion.cc
+        fam/FamRecordVersion.h
+        fam/FamIndex.cc
+        fam/FamIndex.h
+        fam/FamStore.cc
+        fam/FamStore.h
     )
     set( fdb5_srcs  ${fdb5_srcs} PARENT_SCOPE )
 endif()

src/fdb5/fam/FamCatalogue.cc

@@ -0,0 +1,143 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+#include "fdb5/fam/FamCatalogue.h"
+
+#include "eckit/filesystem/URI.h"
+// #include "eckit/log/Timer.h"
+#include "fdb5/LibFdb5.h"
+// #include "fdb5/fam/FamMoveVisitor.h"
+// #include "fdb5/fam/FamPurgeVisitor.h"
+// #include "fdb5/fam/FamStats.h"
+// #include "fdb5/fam/FamWipeVisitor.h"
+// #include "fdb5/fam/RootManager.h"
+#include "fdb5/rules/Rule.h"
+
+// using namespace eckit;
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+FamCatalogue::FamCatalogue(const eckit::FamRegionName& region, const ControlIdentifiers& identifiers, const Config& config):
+    FamCommon(region),
+    Catalogue(Key(), identifiers, config),
+    version_ {FamRecordVersion(config)},
+    table_ {root_.object("table")} { }
+
+FamCatalogue::FamCatalogue(const eckit::URI& uri, const Config& config):
+    FamCatalogue(uri, ControlIdentifier {}, config) { }
+
+// FamCatalogue::FamCatalogue(const Key& key, const fdb5::Config& config):
+//     FamCommon(config, key), Catalogue(key, ControlIdentifier {}, config) { }
+
+// FamCatalogue::FamCatalogue(const eckit::PathName&    directory,
+//                            const ControlIdentifiers& controlIdentifiers,
+//                            const fdb5::Config&       config):
+//     Catalogue(Key(), controlIdentifiers, config), FamHandler(directory, config) {
+//     // Read the real DB key into the DB base object
+//     dbKey_ = databaseKey();
+// }
+
+//----------------------------------------------------------------------------------------------------------------------
+
+void FamCatalogue::remove(const eckit::FamObjectName& name, std::ostream& logAlways, std::ostream& logVerbose, bool doit) {
+    logVerbose << "remove: ";
+    logAlways << name.uri() << '\n';
+    // if (doit) { name.lookup().deallocate(); }
+    NOTIMP;
+}
+
+auto FamCatalogue::uri() const -> eckit::URI {
+    return FamCommon::uri();
+}
+
+auto FamCatalogue::exists() const -> bool {
+    return FamCommon::exists();
+}
+
+void FamCatalogue::loadSchema() {
+    NOTIMP;
+    // const auto schema = root_.object("schema").lookup().buffer();
+    // std::stringstream stream;
+    // stream << schema.view();
+    // schema_.load(stream, true);
+}
+
+auto FamCatalogue::schema() const -> const Schema& {
+    return schema_;
+}
+
+auto FamCatalogue::indexes(const bool /* sorted */) const -> std::vector<Index> {
+    std::vector<Index> result;
+
+    /// @todo here would be the eckit::FamMap as KV store on FAM, which is not implemented yet
+    NOTIMP;
+
+    return result;
+}
+
+void FamCatalogue::dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const {
+    NOTIMP;
+    // const bool walkSubToc = conf.getBool(DUMP_PARAM_WALK, false);
+    // FamHandler::dump(out, simple, walkSubToc);
+}
+
+void FamCatalogue::maskIndexEntry(const Index& index) const {
+    NOTIMP;
+    // FamHandler(root_, config_).writeClearRecord(index);
+}
+
+auto FamCatalogue::metadataPaths() const -> std::vector<eckit::PathName> {
+    NOTIMP;
+}
+
+StatsReportVisitor* FamCatalogue::statsReportVisitor() const {
+    NOTIMP;
+}
+
+PurgeVisitor* FamCatalogue::purgeVisitor(const Store& /* store */) const {
+    NOTIMP;
+}
+
+WipeVisitor* FamCatalogue::wipeVisitor(const Store&                     store,
+                                       const metkit::mars::MarsRequest& request,
+                                       std::ostream&                    out,
+                                       bool                             doit,
+                                       bool                             porcelain,
+                                       bool                             unsafeWipeAll) const {
+    NOTIMP;
+}
+
+MoveVisitor* FamCatalogue::moveVisitor(const Store&                     store,
+                                       const metkit::mars::MarsRequest& request,
+                                       const eckit::URI&                dest,
+                                       eckit::Queue<MoveElement>&       queue) const {
+    NOTIMP;
+}
+
+void FamCatalogue::allMasked(std::set<std::pair<eckit::URI, eckit::Offset>>& /* metadata */,
+                             std::set<eckit::URI>& /* data */) const {
+    NOTIMP;
+}
+
+void FamCatalogue::control(const ControlAction& /* action */, const ControlIdentifiers& /* identifiers */) const {
+    NOTIMP;
+    // FamHandler::control(action, identifiers);
+}
+
+bool FamCatalogue::enabled(const ControlIdentifier& /* identifier */) const {
+    NOTIMP;
+    // return Catalogue::enabled(identifier) && FamHandler::enabled(identifier);
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace fdb5

src/fdb5/fam/FamCatalogue.h

@@ -0,0 +1,114 @@
+/*
+ * (C) Copyright 1996- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+
+/*
+ * This software was developed as part of the Horizon Europe programme funded project OpenCUBE
+ * (Grant agreement: 101092984) horizon-opencube.eu
+ */
+
+/// @file   FamCatalogue.h
+/// @author Metin Cakircali
+/// @date   Jul 2024
+
+#pragma once
+
+#include "eckit/io/fam/FamObjectName.h"
+#include "eckit/io/fam/FamPath.h"
+#include "fdb5/database/Index.h"
+#include "fdb5/fam/FamCommon.h"
+#include "fdb5/fam/FamEngine.h"
+#include "fdb5/fam/FamHandler.h"
+#include "fdb5/rules/Schema.h"
+
+namespace fdb5 {
+
+//----------------------------------------------------------------------------------------------------------------------
+
+/// @brief FDB Catalogue for Fabric Attached Memory systems
+// class FamCatalogue: protected FamCommon, public Catalogue, public FamHandler {
+class FamCatalogue: protected FamCommon, public Catalogue {
+    friend class FamWipeVisitor;
+    friend class FamMoveVisitor;
+
+public:  // types
+    using FamCommon::typeName;
+
+    static constexpr auto DUMP_PARAM_WALK = "walk";
+
+public:  // methods
+    FamCatalogue(const eckit::FamRegionName& region, const ControlIdentifiers& identifiers, const Config& config);
+
+    FamCatalogue(const eckit::URI& uri, const Config& config);
+
+    // FamCatalogue(const Key& key, const Config& config);
+
+    ~FamCatalogue() = default;
+
+    // static auto catalogueTypeName() -> const char* { return typeName; }
+
+    static void remove(const eckit::FamObjectName& name, std::ostream& logAlways, std::ostream& logVerbose, bool doit);
+
+    // const eckit::PathName& basePath() const override;
+    auto uri() const -> eckit::URI override;
+
+    auto indexKey() const -> const Key& override { return indexKey_; }
+
+    auto enabled(const ControlIdentifier& identifier) const -> bool override;
+
+protected:  // methods
+    auto type() const -> std::string override { return typeName; }
+
+    bool exists() const override;
+
+    void loadSchema() override;
+
+    auto schema() const -> const Schema& override;
+
+    auto indexes(bool sorted = false) const -> std::vector<Index> override;
+
+    void dump(std::ostream& out, bool simple, const eckit::Configuration& conf) const override;
+
+    auto metadataPaths() const -> std::vector<eckit::PathName> override;
+
+    auto statsReportVisitor() const -> StatsReportVisitor* override;
+
+    auto purgeVisitor(const Store& store) const -> PurgeVisitor* override;
+
+    auto wipeVisitor(const Store&                     store,
+                     const metkit::mars::MarsRequest& request,
+                     std::ostream&                    out,
+                     bool                             doit,
+                     bool                             porcelain,
+                     bool                             unsafe) const -> WipeVisitor* override;
+
+    auto moveVisitor(const Store&                     store,
+                     const metkit::mars::MarsRequest& request,
+                     const eckit::URI&                dest,
+                     eckit::Queue<MoveElement>&       queue) const -> MoveVisitor* override;
+
+    void maskIndexEntry(const Index& index) const override;
+
+    void allMasked(std::set<std::pair<eckit::URI, eckit::Offset>>& metadata, std::set<eckit::URI>& data) const override;
+
+    void control(const ControlAction& action, const ControlIdentifiers& identifiers) const override;
+
+private:  // members
+    const FamRecordVersion version_;
+
+    eckit::FamObjectName table_;
+
+    Schema schema_;
+
+    Key indexKey_;
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+
+}  // namespace fdb5