Merge branch 'develop' into remoteFDB

ecmwf · Nov 6, 2024 · 07dc074 · 07dc074
2 parents e32c79d + 72fd6af
commit 07dc074
Show file tree

Hide file tree

Showing 22 changed files with 38 additions and 148 deletions.
diff --git a/src/fdb5/tools/FDBLock.cc b/src/fdb5/tools/FDBLock.cc
@@ -12,6 +12,7 @@
 
 #include "eckit/log/Log.h"
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"

diff --git a/src/fdb5/tools/FDBTool.cc b/src/fdb5/tools/FDBTool.cc
@@ -9,14 +9,11 @@
  */
 
 #include "eckit/option/CmdArgs.h"
-#include "eckit/types/Date.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/LibFdb5.h"
-#include "fdb5/rules/Schema.h"
 #include "fdb5/tools/FDBTool.h"
 
-using eckit::Log;
-
 namespace fdb5 {
 
 //----------------------------------------------------------------------------------------------------------------------
@@ -34,7 +31,9 @@ static void usage(const std::string& tool) {
 }
 
 void FDBTool::run() {
-    options_.push_back(new eckit::option::SimpleOption<std::string>("config", "FDB configuration filename"));
+    if(needsConfig_) {
+        options_.push_back(new eckit::option::SimpleOption<std::string>("config", "FDB configuration filename"));
+    }
 
     eckit::option::CmdArgs args(&fdb5::usage, options_, numberOfPositionalArguments(),
                                 minimumPositionalArguments());

diff --git a/src/fdb5/tools/FDBTool.h b/src/fdb5/tools/FDBTool.h
@@ -19,10 +19,11 @@
 #include <vector>
 
 #include "eckit/runtime/Tool.h"
-#include "eckit/filesystem/PathName.h"
+#include "eckit/config/Configuration.h"
+#include "eckit/config/LocalConfiguration.h"
+#include "eckit/exception/Exceptions.h"
 
-#include "fdb5/database/Catalogue.h"
-#include "eckit/option/SimpleOption.h"
+#include "fdb5/config/Config.h"
 
 namespace eckit {
     namespace option {
@@ -55,6 +56,8 @@ class FDBTool : public eckit::Tool {
 protected: // members
 
     std::vector<eckit::option::Option *> options_;
+    /// Set this to false in tool subclass if your tool does not require access to 'config.yaml'
+    bool needsConfig_{true};
 
 protected: // methods
 

diff --git a/src/fdb5/tools/fdb-copy.cc b/src/fdb5/tools/fdb-copy.cc
@@ -11,8 +11,8 @@
 #include <fstream>
 #include <memory>
 
-#include "eckit/io/FileHandle.h"
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 #include "eckit/filesystem/PathName.h"
 
 #include "metkit/mars/MarsRequest.h"

diff --git a/src/fdb5/tools/fdb-dump-index.cc b/src/fdb5/tools/fdb-dump-index.cc
@@ -11,23 +11,19 @@
 #include "eckit/option/CmdArgs.h"
 
 #include "fdb5/toc/TocHandler.h"
-#include "fdb5/toc/TocIndex.h"
 #include "fdb5/tools/FDBTool.h"
 
-#include "fdb5/toc/BTreeIndex.h"
-#include "fdb5/database/Index.h"
-
 #include <string>
 
 using namespace eckit;
 
 //----------------------------------------------------------------------------------------------------------------------
 
-class FDBDumpToc : public fdb5::FDBTool {
+class FDBDumpIndex : public fdb5::FDBTool {
 
   public: // methods
 
-    FDBDumpToc(int argc, char **argv) :
+    FDBDumpIndex(int argc, char **argv) :
         fdb5::FDBTool(argc, argv) {}
 
   private: // methods
@@ -36,14 +32,14 @@ class FDBDumpToc : public fdb5::FDBTool {
     virtual void execute(const option::CmdArgs& args);
 };
 
-void FDBDumpToc::usage(const std::string &tool) const {
+void FDBDumpIndex::usage(const std::string &tool) const {
     Log::info() << std::endl
                 << "Usage: " << tool << " [path1] [path2] ..." << std::endl;
     fdb5::FDBTool::usage(tool);
 }
 
 
-void FDBDumpToc::execute(const option::CmdArgs& args) {
+void FDBDumpIndex::execute(const option::CmdArgs& args) {
 
     // n.b. We don't just open the toc, then check if in the list of indexes, as there
     //      is no reason to think that the indexes map to the same toc (or directory).
@@ -62,7 +58,7 @@ void FDBDumpToc::execute(const option::CmdArgs& args) {
 //----------------------------------------------------------------------------------------------------------------------
 
 int main(int argc, char **argv) {
-    FDBDumpToc app(argc, argv);
+    FDBDumpIndex app(argc, argv);
     return app.start();
 }
 
diff --git a/src/fdb5/tools/fdb-dump-toc.cc b/src/fdb5/tools/fdb-dump-toc.cc
@@ -9,6 +9,7 @@
  */
 
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/toc/TocHandler.h"
 #include "fdb5/tools/FDBTool.h"
@@ -21,9 +22,9 @@ class FDBDumpToc : public fdb5::FDBTool {
 
   public: // methods
 
-    FDBDumpToc(int argc, char **argv) :
-        fdb5::FDBTool(argc, argv) {
-
+    FDBDumpToc(int argc, char **argv) : fdb5::FDBTool(argc, argv) {
+        // FDBDumpToc does not require to read the configuration
+        needsConfig_ = false;
         options_.push_back(new eckit::option::SimpleOption<bool>("walk", "Walk subtocs rather than show simple entries"));
     }
 

diff --git a/src/fdb5/tools/fdb-dump.cc b/src/fdb5/tools/fdb-dump.cc
@@ -13,8 +13,6 @@
 
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
-#include "fdb5/database/Index.h"
-#include "fdb5/rules/Schema.h"
 #include "fdb5/tools/FDBVisitTool.h"
 
 using namespace eckit;

diff --git a/src/fdb5/tools/fdb-hammer.cc b/src/fdb5/tools/fdb-hammer.cc
@@ -26,7 +26,6 @@
 #include "eckit/io/EmptyHandle.h"
 #include "eckit/option/CmdArgs.h"
 #include "eckit/option/SimpleOption.h"
-#include "eckit/option/VectorOption.h"
 
 #include "fdb5/message/MessageArchiver.h"
 #include "fdb5/io/HandleGatherer.h"
@@ -354,7 +353,7 @@ void FDBHammer::executeList(const eckit::option::CmdArgs &args) {
         number_values.push_back(std::to_string(n + number - 1));
     }
     request.values("number", number_values);
-   
+
     std::vector<std::string> levelist_values;
     for (size_t l = 1; l <= nlevels; ++l) {
         levelist_values.push_back(std::to_string(l + level - 1));
@@ -373,7 +372,7 @@ void FDBHammer::executeList(const eckit::option::CmdArgs &args) {
 
     size_t count = 0;
     for (size_t step = 0; step < nsteps; ++step) {
- 
+
         request.setValue("step", step);
 
         auto listObject = fdb.list(fdb5::FDBToolRequest(request, false, minimumKeys));

diff --git a/src/fdb5/tools/fdb-hide.cc b/src/fdb5/tools/fdb-hide.cc
@@ -8,18 +8,15 @@
  * does it submit to any jurisdiction.
  */
 
-#include "eckit/config/Resource.h"
 #include "eckit/option/CmdArgs.h"
-#include "eckit/option/VectorOption.h"
-#include "eckit/os/AutoUmask.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Schema.h"
 #include "fdb5/toc/TocCatalogueWriter.h"
-#include "fdb5/toc/TocCatalogueReader.h"
 #include "fdb5/toc/TocEngine.h"
 #include "fdb5/tools/FDBTool.h"
 

diff --git a/src/fdb5/tools/fdb-info.cc b/src/fdb5/tools/fdb-info.cc
@@ -9,15 +9,12 @@
  */
 
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/LibFdb5.h"
+#include "fdb5/tools/FDBTool.h"
 #include "fdb5/config/Config.h"
-#include "fdb5/database/Catalogue.h"
-#include "fdb5/database/Index.h"
-#include "fdb5/tools/FDBInspect.h"
 #include "fdb5/io/LustreSettings.h"
-
-#include "fdb5/fdb5_config.h"
 #include "fdb5/fdb5_version.h"
 
 using eckit::Log;

diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc
@@ -11,17 +11,14 @@
 #include <unordered_set>
 
 #include "eckit/option/CmdArgs.h"
-#include "eckit/config/Resource.h"
 #include "eckit/option/SimpleOption.h"
-#include "eckit/option/VectorOption.h"
 #include "eckit/option/CmdArgs.h"
 #include "eckit/log/JSON.h"
 
 #include "metkit/hypercube/HyperCube.h"
 
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
-#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/rules/Schema.h"
 #include "fdb5/tools/FDBVisitTool.h"
@@ -104,13 +101,13 @@ void FDBList::init(const CmdArgs& args) {
     if (compact_) {
         if (location_) {
             throw UserError("--compact and --location are not compatible", Here());
-        } 
+        }
         if (full_) {
             throw UserError("--compact and --full are not compatible", Here());
-        } 
+        }
         if (porcelain_) {
             throw UserError("--compact and --porcelain are not compatible", Here());
-        } 
+        }
     }
 
     /// @todo option ignore-errors

diff --git a/src/fdb5/tools/fdb-move.cc b/src/fdb5/tools/fdb-move.cc
@@ -22,7 +22,6 @@
 #include "fdb5/api/FDB.h"
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/LibFdb5.h"
-#include "fdb5/toc/TocCommon.h"
 
 #define MAX_THREADS 256
 
@@ -163,7 +162,7 @@ class MoveProducer : public eckit::distributed::Producer {
             }
         }
     }
-    
+
     void messageFromWorker( eckit::distributed::Message& message, int worker) const {}
 
 private: // attributes

diff --git a/src/fdb5/tools/fdb-overlay.cc b/src/fdb5/tools/fdb-overlay.cc
@@ -8,17 +8,15 @@
  * does it submit to any jurisdiction.
  */
 
-#include "eckit/config/Resource.h"
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 #include "eckit/option/VectorOption.h"
 
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/LibFdb5.h"
 #include "fdb5/rules/Schema.h"
-/*#include "fdb5/toc/TocCatalogueWriter.h"
-#include "fdb5/toc/TocDBReader.h"*/
 #include "fdb5/toc/TocEngine.h"
 #include "fdb5/tools/FDBTool.h"
 

diff --git a/src/fdb5/tools/fdb-patch.cc b/src/fdb5/tools/fdb-patch.cc
@@ -12,7 +12,6 @@
 
 #include "eccodes.h"
 
-#include "eckit/config/Resource.h"
 #include "eckit/option/CmdArgs.h"
 #include "eckit/option/SimpleOption.h"
 #include "eckit/log/Bytes.h"

diff --git a/src/fdb5/tools/fdb-read.cc b/src/fdb5/tools/fdb-read.cc
@@ -13,6 +13,7 @@
 
 #include "eckit/io/FileHandle.h"
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "metkit/mars/MarsRequest.h"
 #include "metkit/mars/MarsParser.h"

diff --git a/src/fdb5/tools/fdb-reconsolidate-toc.cc b/src/fdb5/tools/fdb-reconsolidate-toc.cc
@@ -9,10 +9,11 @@
  */
 
 #include "fdb5/tools/FDBTool.h"
+#include "fdb5/database/DB.h"
 
 #include "eckit/option/CmdArgs.h"
 #include "eckit/config/LocalConfiguration.h"
-#include "fdb5/database/Catalogue.h"
+#include "eckit/filesystem/URI.h"
 
 using namespace eckit;
 
@@ -53,8 +54,7 @@ void FDBReconsolidateToc::execute(const eckit::option::CmdArgs& args) {
         dbPath = dbPath.dirName();
     }
 
-    // TODO: In updated version, grab default Config() here;
-    std::unique_ptr<fdb5::CatalogueWriter> catalogue = fdb5::CatalogueWriterFactory::instance().build(eckit::URI("toc", dbPath), eckit::LocalConfiguration());
+    std::unique_ptr<fdb5::CatalogueWriter> catalogue = fdb5::CatalogueWriterFactory::instance().build(eckit::URI("toc", dbPath), config(args));
 
     catalogue->reconsolidate();
 }

diff --git a/src/fdb5/tools/fdb-root.cc b/src/fdb5/tools/fdb-root.cc
@@ -8,8 +8,8 @@
  * does it submit to any jurisdiction.
  */
 
-#include "eckit/config/Resource.h"
 #include "eckit/option/CmdArgs.h"
+#include "eckit/option/SimpleOption.h"
 
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/config/Config.h"