From 55be3ded7e6841179932aa2877927864f8dcb0e5 Mon Sep 17 00:00:00 2001 From: Simon Smart Date: Tue, 27 Feb 2024 23:05:10 +0000 Subject: [PATCH 1/5] FDB-321: Add axes() support --- src/fdb5/CMakeLists.txt | 5 + src/fdb5/api/FDB.cc | 10 ++ src/fdb5/api/FDB.h | 3 + src/fdb5/api/FDBFactory.h | 4 +- src/fdb5/api/LocalFDB.cc | 6 + src/fdb5/api/LocalFDB.h | 2 + src/fdb5/api/helpers/AxesIterator.cc | 41 +++++++ src/fdb5/api/helpers/AxesIterator.h | 74 +++++++++++++ src/fdb5/api/local/AxesVisitor.cc | 82 ++++++++++++++ src/fdb5/api/local/AxesVisitor.h | 57 ++++++++++ src/fdb5/database/IndexAxis.cc | 66 ++++++++++- src/fdb5/database/IndexAxis.h | 16 ++- src/fdb5/toc/TocCatalogue.cc | 4 +- src/fdb5/tools/fdb-axes.cc | 83 ++++++++++++++ tests/fdb/CMakeLists.txt | 1 + tests/fdb/database/CMakeLists.txt | 9 ++ tests/fdb/database/test_indexaxis.cc | 157 +++++++++++++++++++++++++++ tests/fdb/tools/CMakeLists.txt | 3 +- tests/fdb/tools/fdb_axes.sh.in | 68 ++++++++++++ tests/fdb/tools/local.yaml | 6 + tests/fdb/tools/x.grib | 1 + 21 files changed, 691 insertions(+), 7 deletions(-) create mode 100644 src/fdb5/api/helpers/AxesIterator.cc create mode 100644 src/fdb5/api/helpers/AxesIterator.h create mode 100644 src/fdb5/api/local/AxesVisitor.cc create mode 100644 src/fdb5/api/local/AxesVisitor.h create mode 100644 src/fdb5/tools/fdb-axes.cc create mode 100644 tests/fdb/database/CMakeLists.txt create mode 100644 tests/fdb/database/test_indexaxis.cc create mode 100755 tests/fdb/tools/fdb_axes.sh.in create mode 100644 tests/fdb/tools/local.yaml create mode 120000 tests/fdb/tools/x.grib diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 8ccf68784..525cf8837 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -32,6 +32,8 @@ list( APPEND fdb5_srcs api/SelectFDB.h api/helpers/APIIterator.h + api/helpers/Axesiterator.cc + api/helpers/AxesIterator.h api/helpers/ControlIterator.cc api/helpers/ControlIterator.h api/helpers/FDBToolRequest.cc @@ -50,6 +52,8 @@ list( APPEND fdb5_srcs api/local/QueryVisitor.h api/local/QueueStringLogTarget.h api/local/ListVisitor.h + api/local/AxesVisitor.cc + api/local/AxesVisitor.h api/local/ControlVisitor.cc api/local/ControlVisitor.h api/local/DumpVisitor.h @@ -408,6 +412,7 @@ if(HAVE_FDB_BUILD_TOOLS) endif() list( APPEND fdb5_tools + fdb-axes fdb-write fdb-copy fdb-dump diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index f04547e36..7ff05a04c 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -294,6 +294,16 @@ void FDB::flush() { } } +IndexAxis FDB::axes(const FDBToolRequest& request, int level) { + IndexAxis axes; + AxesElement elem; + auto it = internal_->axes(request, level); + while (it.next(elem)) { + axes.merge(elem.axes()); + } + return axes; +} + bool FDB::dirty() const { return dirty_; } diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index cb9879363..54dab0c52 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -109,6 +109,9 @@ class FDB { ControlIterator control(const FDBToolRequest& request, ControlAction action, ControlIdentifiers identifiers); + + IndexAxis axes(const FDBToolRequest& request, int level=3); + bool enabled(const ControlIdentifier& controlIdentifier) const; bool dirty() const; diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h index 1d9970d5f..2f0f6a62d 100644 --- a/src/fdb5/api/FDBFactory.h +++ b/src/fdb5/api/FDBFactory.h @@ -28,7 +28,7 @@ #include "fdb5/database/DB.h" #include "fdb5/config/Config.h" #include "fdb5/api/FDBStats.h" -#include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/api/helpers/AxesIterator.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/api/helpers/ControlIterator.h" #include "fdb5/api/helpers/DumpIterator.h" @@ -88,6 +88,8 @@ class FDBBase : private eckit::NonCopyable { virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0; + virtual AxesIterator axes(const FDBToolRequest& request, int axes) { NOTIMP; } + // -------------- API management ---------------------------- /// ID used for hashing in the Rendezvous hash. Should be unique amongst those used diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc index b2ea1a7ea..8705a6da4 100644 --- a/src/fdb5/api/LocalFDB.cc +++ b/src/fdb5/api/LocalFDB.cc @@ -29,6 +29,7 @@ #include "fdb5/rules/Schema.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/local/AxesVisitor.h" #include "fdb5/api/local/ControlVisitor.h" #include "fdb5/api/local/DumpVisitor.h" #include "fdb5/api/local/ListVisitor.h" @@ -123,6 +124,11 @@ ControlIterator LocalFDB::control(const FDBToolRequest& request, return queryInternal(request, action, identifiers); } +AxesIterator LocalFDB::axes(const FDBToolRequest& request, int level) { + LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::axes() : " << request << std::endl; + return queryInternal(request, config_, level); +} + void LocalFDB::flush() { if (archiver_) { archiver_->flush(); diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h index e7b226c78..96f4a6914 100644 --- a/src/fdb5/api/LocalFDB.h +++ b/src/fdb5/api/LocalFDB.h @@ -59,6 +59,8 @@ class LocalFDB : public FDBBase { MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) override; + AxesIterator axes(const FDBToolRequest& request, int axes) override; + void flush() override; private: // methods diff --git a/src/fdb5/api/helpers/AxesIterator.cc b/src/fdb5/api/helpers/AxesIterator.cc new file mode 100644 index 000000000..99cdfe89a --- /dev/null +++ b/src/fdb5/api/helpers/AxesIterator.cc @@ -0,0 +1,41 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project NextGenIO + * (Project ID: 671951) www.nextgenio.eu + */ + +#include "fdb5/api/helpers/AxesIterator.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +AxesElement::AxesElement(Key&& dbKey, IndexAxis&& axes) : + dbKey_(std::move(dbKey)), axes_(std::move(axes)) {} + +AxesElement::AxesElement(eckit::Stream& s) { + s >> dbKey_; + axes_ = IndexAxis(s, IndexAxis::currentVersion()); +} + +void AxesElement::print(std::ostream& out) const { + out << "Axes(db=" << dbKey_ << ", axes=" << axes_ << ")"; +} + +void AxesElement::encode(eckit::Stream& s) const { + s << dbKey_; + axes_.encode(s, IndexAxis::currentVersion()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/api/helpers/AxesIterator.h b/src/fdb5/api/helpers/AxesIterator.h new file mode 100644 index 000000000..9881358dd --- /dev/null +++ b/src/fdb5/api/helpers/AxesIterator.h @@ -0,0 +1,74 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Simon Smart +/// @date January 2024 + +#pragma once + +#include "fdb5/database/Key.h" +#include "fdb5/database/IndexAxis.h" +#include "fdb5/api/helpers/APIIterator.h" + +namespace eckit { +class Stream; +class JSON; +} + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class AxesElement { +public: // methods + + AxesElement() = default; + AxesElement(Key&& dbKey, IndexAxis&& axis); + explicit AxesElement(eckit::Stream& s); + + [[ nodiscard ]] + const Key& key() const { return dbKey_; } + + [[ nodiscard ]] + const IndexAxis& axes() const { return axes_; } + + void print(std::ostream& out) const; + +private: // methods + + void encode(eckit::Stream& s) const; + + friend std::ostream& operator<<(std::ostream& os, const AxesElement& e) { + e.print(os); + return os; + } + + friend eckit::Stream& operator<<(eckit::Stream& s, const AxesElement& r) { + r.encode(s); + return s; + } + +private: // members + + Key dbKey_; + IndexAxis axes_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +using AxesAggregateIterator = APIAggregateIterator; + +using AxesAsyncIterator = APIAsyncIterator; + +using AxesIterator = APIIterator; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/api/local/AxesVisitor.cc b/src/fdb5/api/local/AxesVisitor.cc new file mode 100644 index 000000000..a1a2a536a --- /dev/null +++ b/src/fdb5/api/local/AxesVisitor.cc @@ -0,0 +1,82 @@ +/* + * (C) Copyright 2018- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/api/local/AxesVisitor.h" + +#include "fdb5/database/Catalogue.h" + +namespace fdb5 { +namespace api { +namespace local { + +//---------------------------------------------------------------------------------------------------------------------- + +AxesVisitor::AxesVisitor(eckit::Queue& queue, + const metkit::mars::MarsRequest& request, + const Config& config, + int level) : + QueryVisitor(queue, request), + schema_(config.schema()), + level_(level) {} + +#if 0 + +// TODO: Here we can do nice tricks to make things go muuuuuuuuuuuuuch faster... +// See improvements to the EntryVisitMechanism... & the schema + +bool AxesVisitor::preVisitDatabase(const eckit::URI& uri) { + + // If level == 1, avoid constructing the Catalogue/Store objects, so just interrogate the URIs + if (level_ == 1 && uri.scheme() == "toc") { + // TODO: This is hacky, only works with the toc backend... + if (schema_.matchFirstLevel(uri.path().baseName(), dbKey_)) { + axes_.wipe(); + axes_.insert(dbKey_); + axes_.sort(); + queue_.emplace(AxesElement{std::move(dbKey_), std::move(axes_)}); + } + return false; + } + + return true; +} +#endif + +bool AxesVisitor::visitDatabase(const Catalogue& catalogue, const Store& store) { + dbKey_ = catalogue.key(); + axes_.wipe(); + axes_.insert(dbKey_); + axes_.sort(); + return (level_ > 1); +} + +bool AxesVisitor::visitIndex(const Index& index) { + if (index.partialMatch(request_)) { + IndexAxis tmpAxis; + tmpAxis.insert(index.key()); + tmpAxis.sort(); + axes_.merge(tmpAxis); // avoid sorts on the (growing) main Axes object + + if (level_ > 2) { + axes_.merge(index.axes()); + } + } + return false; +} + +void AxesVisitor::catalogueComplete(const fdb5::Catalogue& catalogue) { + queue_.emplace(AxesElement{std::move(dbKey_), std::move(axes_)}); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace local +} // namespace api +} // namespace fdb5 diff --git a/src/fdb5/api/local/AxesVisitor.h b/src/fdb5/api/local/AxesVisitor.h new file mode 100644 index 000000000..e2a3d8987 --- /dev/null +++ b/src/fdb5/api/local/AxesVisitor.h @@ -0,0 +1,57 @@ +/* + * (C) Copyright 2018- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Simon Smart +/// @date January 2024 + +#pragma once + +#include "fdb5/api/local/QueryVisitor.h" +#include "fdb5/api/helpers/AxesIterator.h" + + +namespace fdb5 { +namespace api { +namespace local { + +/// @note Helper classes for LocalFDB + +//---------------------------------------------------------------------------------------------------------------------- + +class AxesVisitor : public QueryVisitor { +public: + + AxesVisitor(eckit::Queue& queue, + const metkit::mars::MarsRequest& request, + const Config& config, + int level); + + bool visitIndexes() override { return true; } + bool visitEntries() override { return false; } + void catalogueComplete(const fdb5::Catalogue& catalogue) override; + +// bool preVisitDatabase(const eckit::URI& uri) override; + bool visitDatabase(const Catalogue& catalogue, const Store& store) override; + bool visitIndex(const Index&) override; + void visitDatum(const Field&, const Key&) override { NOTIMP; } + +private: // members + + Key dbKey_; + IndexAxis axes_; + const Schema& schema_; + int level_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace local +} // namespace api +} // namespace fdb5 diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc index 912b49be7..aec44d4e9 100755 --- a/src/fdb5/database/IndexAxis.cc +++ b/src/fdb5/database/IndexAxis.cc @@ -45,6 +45,35 @@ IndexAxis::IndexAxis(eckit::Stream &s, const int version) : decode(s, version); } +IndexAxis::IndexAxis(IndexAxis&& rhs) noexcept : + axis_(std::move(rhs.axis_)), + readOnly_(rhs.readOnly_), + dirty_(rhs.dirty_) {} + +IndexAxis& IndexAxis::operator=(IndexAxis&& rhs) noexcept { + axis_ = std::move(rhs.axis_); + readOnly_ = rhs.readOnly_; + dirty_ = rhs.dirty_; + return *this; +} + +bool IndexAxis::operator==(const IndexAxis& rhs) const { + + if (axis_.size() != rhs.axis_.size()) return false; + + for (const auto& kv : axis_) { + auto it = rhs.axis_.find(kv.first); + if (it == rhs.axis_.end()) return false; + if (*kv.second != *it->second) return false; + } + + return true; +} + +bool IndexAxis::operator!=(const IndexAxis& rhs) const { + return !(*this == rhs); +} + void IndexAxis::encode(eckit::Stream &s, const int version) const { if (version >= 3) { encodeCurrent(s, version); @@ -295,10 +324,45 @@ const eckit::DenseSet &IndexAxis::values(const std::string &keyword void IndexAxis::print(std::ostream &out) const { out << "IndexAxis[" << "axis="; - eckit::__print_container(out, axis_); + + const char* sep = ""; + out << "{"; + for (const auto& kv : axis_) { + out << sep << kv.first << "=("; + const char* sep2 = ""; + for (const auto& v : *kv.second) { + out << sep2 << v; + sep2 = ","; + } + out << ")"; + sep = ","; + } + out << "}"; out << "]"; } +void IndexAxis::json(eckit::JSON& json) const { + json.startObject(); + for (const auto& kv : axis_) { + json << kv.first << *kv.second; + } + json.endObject(); +} + +void IndexAxis::merge(const fdb5::IndexAxis& other) { + + ASSERT(!readOnly_); + for (const auto& kv : other.axis_) { + + auto it = axis_.find(kv.first); + if (it == axis_.end()) { + axis_.emplace(kv.first, kv.second); + } else { + it->second->merge(*kv.second); + }; + } +} + //---------------------------------------------------------------------------------------------------------------------- } // namespace fdb5 diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h index a83b92b6d..e581dbe3b 100644 --- a/src/fdb5/database/IndexAxis.h +++ b/src/fdb5/database/IndexAxis.h @@ -45,11 +45,20 @@ class IndexAxis : private eckit::NonCopyable { IndexAxis(); IndexAxis(eckit::Stream &s, const int version); + IndexAxis(IndexAxis&& rhs) noexcept; + + IndexAxis& operator=(IndexAxis&& rhs) noexcept; ~IndexAxis(); + bool operator==(const IndexAxis& rhs) const; + bool operator!=(const IndexAxis& rhs) const; + void insert(const Key &key); void encode(eckit::Stream &s, const int version) const; + static int currentVersion() { return 3; } + + void merge(const IndexAxis& other); // Decode can be used for two-stage initialisation (IndexAxis a; a.decode(s);) void decode(eckit::Stream& s, const int version); @@ -78,6 +87,11 @@ class IndexAxis : private eckit::NonCopyable { return s; } + friend eckit::JSON& operator<<(eckit::JSON& j, const IndexAxis& x) { + x.json(j); + return j; + } + private: // methods void encodeCurrent(eckit::Stream &s, const int version) const; @@ -87,7 +101,7 @@ class IndexAxis : private eckit::NonCopyable { void decodeLegacy(eckit::Stream& s, const int version); void print(std::ostream &out) const; - + void json(eckit::JSON& j) const; private: // members diff --git a/src/fdb5/toc/TocCatalogue.cc b/src/fdb5/toc/TocCatalogue.cc index 9241d1eb4..63188fd83 100644 --- a/src/fdb5/toc/TocCatalogue.cc +++ b/src/fdb5/toc/TocCatalogue.cc @@ -92,10 +92,8 @@ void TocCatalogue::visitEntries(EntryVisitor& visitor, const Store& store, bool } } } - - visitor.catalogueComplete(*this); } - + visitor.catalogueComplete(*this); } void TocCatalogue::loadSchema() { diff --git a/src/fdb5/tools/fdb-axes.cc b/src/fdb5/tools/fdb-axes.cc new file mode 100644 index 000000000..ed71f215e --- /dev/null +++ b/src/fdb5/tools/fdb-axes.cc @@ -0,0 +1,83 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "eckit/option/CmdArgs.h" +#include "eckit/option/SimpleOption.h" +#include "eckit/log/JSON.h" + +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/tools/FDBVisitTool.h" + +using namespace eckit; +using namespace option; + + +namespace fdb5::tools { + +//---------------------------------------------------------------------------------------------------------------------- + +class FDBAxisTest : public FDBVisitTool { + +public: // methods + + FDBAxisTest(int argc, char **argv) : + FDBVisitTool(argc, argv, "class,expver"), + level_(3), + json_(false) { + options_.push_back(new SimpleOption("level", "Specify how many levels of the keys should be should be explored")); + options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); + } + +private: // methods + + void execute(const CmdArgs& args) final; + void init(const CmdArgs &args) final; + +private: // members + + int level_; + bool json_; +}; + +void FDBAxisTest::init(const CmdArgs& args) { + FDBVisitTool::init(args); + json_ = args.getBool("json", json_); + level_ = args.getInt("level", level_); +} + +void FDBAxisTest::execute(const CmdArgs& args) { + + FDB fdb(config(args)); + + for (const FDBToolRequest& request : requests()) { +// Timer taxes("axes"); + auto r = fdb.axes(request, level_); +// taxes.stop(); + + if (json_) { + JSON json(Log::info()); + json << r; + } else { + Log::info() << r; + } + Log::info() << std::endl; + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5::tools + +int main(int argc, char **argv) { + fdb5::tools::FDBAxisTest app(argc, argv); + return app.start(); +} + diff --git a/tests/fdb/CMakeLists.txt b/tests/fdb/CMakeLists.txt index 3f9748011..3bb56caa6 100644 --- a/tests/fdb/CMakeLists.txt +++ b/tests/fdb/CMakeLists.txt @@ -72,5 +72,6 @@ endforeach() add_subdirectory( pmem ) add_subdirectory( api ) +add_subdirectory( database ) add_subdirectory( tools ) add_subdirectory( type ) diff --git a/tests/fdb/database/CMakeLists.txt b/tests/fdb/database/CMakeLists.txt new file mode 100644 index 000000000..1bc08100f --- /dev/null +++ b/tests/fdb/database/CMakeLists.txt @@ -0,0 +1,9 @@ + +list( APPEND _test_environment + FDB_HOME=${PROJECT_BINARY_DIR} ) + +ecbuild_add_test( TARGET test_fdb5_database_indexaxis + SOURCES test_indexaxis.cc + INCLUDES ${PMEM_INCLUDE_DIRS} + LIBS fdb5 + ENVIRONMENT "${_test_environment}") diff --git a/tests/fdb/database/test_indexaxis.cc b/tests/fdb/database/test_indexaxis.cc new file mode 100644 index 000000000..190d50d2a --- /dev/null +++ b/tests/fdb/database/test_indexaxis.cc @@ -0,0 +1,157 @@ + + +#include "fdb5/database/IndexAxis.h" +#include "fdb5/database/Key.h" + +#include "eckit/io/Buffer.h" +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/serialisation/ResizableMemoryStream.h" +#include "eckit/testing/Test.h" + +namespace { + +fdb5::Key EXAMPLE_K1{{ + {"class", "od"}, + {"expver", "0001"}, + {"date", "20240223"} +}}; + +fdb5::Key EXAMPLE_K2{{ + {"class", "rd"}, + {"expver", "0001"}, + {"time", "1200"} +}}; + +fdb5::Key EXAMPLE_K3{{ + {"class", "rd"}, + {"expver", "gotx"}, + {"time", "0000"} +}}; + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("Insertion and comparison") { + + fdb5::IndexAxis ia1; + fdb5::IndexAxis ia2; + EXPECT(ia1 == ia2); + EXPECT(!(ia1 != ia2)); + + ia1.insert(EXAMPLE_K1); + EXPECT(!(ia1 == ia2)); + EXPECT(ia1 != ia2); + + ia2.insert(EXAMPLE_K2); + EXPECT(!(ia1 == ia2)); + EXPECT(ia1 != ia2); + + ia1.insert(EXAMPLE_K2); + EXPECT(!(ia1 == ia2)); + EXPECT(ia1 != ia2); + + ia2.insert(EXAMPLE_K1); + + EXPECT(!(ia1 == ia2)); + EXPECT(ia1 != ia2); + ia1.sort(); + ia2.sort(); + EXPECT(ia1 == ia2); + EXPECT(!(ia1 != ia2)); +} + +CASE("iostream and JSON output functions correctly") { + + fdb5::IndexAxis ia; + + { + std::ostringstream ss; + ss << ia; + EXPECT(ss.str() == "IndexAxis[axis={}]"); + } + + { + std::ostringstream ss; + eckit::JSON json(ss); + json << ia; + EXPECT(ss.str() == "{}"); + } + + ia.insert(EXAMPLE_K1); + ia.insert(EXAMPLE_K2); + ia.insert(EXAMPLE_K3); + ia.sort(); + + { + std::ostringstream ss; + ss << ia; + EXPECT(ss.str() == "IndexAxis[axis={class=(od,rd),date=(20240223),expver=(0001,gotx),time=(0000,1200)}]"); + } + + { + std::ostringstream ss; + eckit::JSON json(ss); + json << ia; + EXPECT(ss.str() == "{\"class\":[\"od\",\"rd\"],\"date\":[\"20240223\"],\"expver\":[\"0001\",\"gotx\"],\"time\":[\"0000\",\"1200\"]}"); + } +} + +CASE("serialiastion and deserialisation") { + + fdb5::IndexAxis ia; + ia.insert(EXAMPLE_K1); + ia.insert(EXAMPLE_K2); + ia.insert(EXAMPLE_K3); + ia.sort(); + + eckit::Buffer buf; + { + eckit::ResizableMemoryStream ms(buf); + ia.encode(ms, fdb5::IndexAxis::currentVersion()); + } + + { + eckit::MemoryStream ms(buf); + fdb5::IndexAxis newia(ms, fdb5::IndexAxis::currentVersion()); + EXPECT(ia == newia); + } +} + +CASE("Check that merging works correctly") { + + fdb5::IndexAxis ia1; + ia1.insert(EXAMPLE_K1); + ia1.insert(EXAMPLE_K2); + ia1.sort(); + + fdb5::IndexAxis ia2; + ia2.insert(EXAMPLE_K1); + ia2.insert(EXAMPLE_K3); + ia2.sort(); + + EXPECT(ia1 != ia2); + + fdb5::IndexAxis iatest; + iatest.insert(EXAMPLE_K1); + iatest.insert(EXAMPLE_K2); + iatest.insert(EXAMPLE_K3); + iatest.sort(); + + EXPECT(iatest != ia1); + EXPECT(iatest != ia2); + + ia1.merge(ia2); + + EXPECT(iatest == ia1); + EXPECT(iatest != ia2); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // anonymous namespace + +int main(int argc, char** argv) { + + eckit::Log::info() << ::getenv("FDB_HOME") << std::endl; + + return ::eckit::testing::run_tests(argc, argv); +} diff --git a/tests/fdb/tools/CMakeLists.txt b/tests/fdb/tools/CMakeLists.txt index 88ad0314d..939db703a 100644 --- a/tests/fdb/tools/CMakeLists.txt +++ b/tests/fdb/tools/CMakeLists.txt @@ -1,5 +1,6 @@ list( APPEND fdb_tools_tests - fdb_info ) + fdb_info + fdb_axes ) foreach( _t ${fdb_tools_tests} ) diff --git a/tests/fdb/tools/fdb_axes.sh.in b/tests/fdb/tools/fdb_axes.sh.in new file mode 100755 index 000000000..a9a3abeb2 --- /dev/null +++ b/tests/fdb/tools/fdb_axes.sh.in @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +set -eux + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB_HOME=@PROJECT_BINARY_DIR@ + +srcdir=@CMAKE_CURRENT_SOURCE_DIR@ +bindir=@CMAKE_CURRENT_BINARY_DIR@ +cd $bindir + +mkdir -p axes_test +cd axes_test + +rm -rf localroot || true +mkdir localroot + +for f in local.yaml x.grib +do + cp $srcdir/$f ./ +done + +export FDB5_CONFIG_FILE=local.yaml + +grib_set -s step=6 x.grib 6.grib +grib_set -s step=9 x.grib 9.grib +grib_set -s type=an 6.grib an6.grib +grib_set -s type=an 9.grib an9.grib + +fdb-write 6.grib +fdb-write 9.grib +fdb-write an6.grib +fdb-write an9.grib +grib_ls -m 9.grib + +####### Check lots of content + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g --json > axes.out +echo '{"class":["rd"],"date":["20201102"],"domain":["g"],"expver":["xxxx"],"levelist":[""],"levtype":["sfc"],"param":["166"],"step":["6","9"],"stream":["oper"],"time":["0000"],"type":["an","fc"]}' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g > axes.out +echo 'IndexAxis[axis={class=(rd),date=(20201102),domain=(g),expver=(xxxx),levelist=(),levtype=(sfc),param=(166),step=(6,9),stream=(oper),time=(0000),type=(an,fc)}]' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g --level=2 > axes.out +echo 'IndexAxis[axis={class=(rd),date=(20201102),domain=(g),expver=(xxxx),levtype=(sfc),stream=(oper),time=(0000),type=(an,fc)}]' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g --level=1 > axes.out +echo 'IndexAxis[axis={class=(rd),date=(20201102),domain=(g),expver=(xxxx),stream=(oper),time=(0000)}]' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out + +####### Sub-selection by index + +fdb-axes class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g,step=6 > axes.out +echo 'IndexAxis[axis={class=(rd),date=(20201102),domain=(g),expver=(xxxx),levelist=(),levtype=(sfc),param=(166),step=(6),stream=(oper),time=(0000),type=(an,fc)}]' > expected.out +cat axes.out +cat expected.out +cmp axes.out expected.out diff --git a/tests/fdb/tools/local.yaml b/tests/fdb/tools/local.yaml new file mode 100644 index 000000000..8e43d2e8f --- /dev/null +++ b/tests/fdb/tools/local.yaml @@ -0,0 +1,6 @@ +--- +type: local +engine: toc +spaces: + - roots: + - path: ./localroot diff --git a/tests/fdb/tools/x.grib b/tests/fdb/tools/x.grib new file mode 120000 index 000000000..7d3306ed4 --- /dev/null +++ b/tests/fdb/tools/x.grib @@ -0,0 +1 @@ +../../regressions/FDB-307/x.grib \ No newline at end of file From ab19780a5e416bd399b8fec8f12161f5a74c69bd Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Wed, 3 Apr 2024 06:59:03 +0200 Subject: [PATCH 2/5] version bump --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7a979be0b..dd0ad7ae6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.11.30 +5.12.0 From 861fdac7c2c9d7d9039f2f116321f67ae370edd7 Mon Sep 17 00:00:00 2001 From: Chris Bradley Date: Thu, 4 Apr 2024 17:53:04 +0100 Subject: [PATCH 3/5] Add method for copying the axis map --- src/fdb5/database/IndexAxis.cc | 11 +++++++++++ src/fdb5/database/IndexAxis.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc index aec44d4e9..a5c436f30 100755 --- a/src/fdb5/database/IndexAxis.cc +++ b/src/fdb5/database/IndexAxis.cc @@ -321,6 +321,17 @@ const eckit::DenseSet &IndexAxis::values(const std::string &keyword return *(i->second); } +std::map> IndexAxis::map() const { + + // Make a copy of the axis map + std::map> result; + + for (const auto& kv : axis_) { + result[kv.first] = eckit::DenseSet(*kv.second); + } + return result; +} + void IndexAxis::print(std::ostream &out) const { out << "IndexAxis[" << "axis="; diff --git a/src/fdb5/database/IndexAxis.h b/src/fdb5/database/IndexAxis.h index e581dbe3b..e4fc4aaac 100644 --- a/src/fdb5/database/IndexAxis.h +++ b/src/fdb5/database/IndexAxis.h @@ -66,6 +66,8 @@ class IndexAxis : private eckit::NonCopyable { bool has(const std::string &keyword) const; const eckit::DenseSet &values(const std::string &keyword) const; + std::map> map() const; + void dump(std::ostream &out, const char* indent) const; bool partialMatch(const metkit::mars::MarsRequest& request) const; From 1ec03f5ab5b4b5e8cc55e72fcf2a5e0185cde46b Mon Sep 17 00:00:00 2001 From: Chris Bradley Date: Thu, 4 Apr 2024 19:15:05 +0100 Subject: [PATCH 4/5] Add unit test --- src/fdb5/database/IndexAxis.cc | 2 +- tests/fdb/database/test_indexaxis.cc | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/fdb5/database/IndexAxis.cc b/src/fdb5/database/IndexAxis.cc index a5c436f30..32dee0583 100755 --- a/src/fdb5/database/IndexAxis.cc +++ b/src/fdb5/database/IndexAxis.cc @@ -327,7 +327,7 @@ std::map> IndexAxis::map() const { std::map> result; for (const auto& kv : axis_) { - result[kv.first] = eckit::DenseSet(*kv.second); + result.emplace(kv.first, *kv.second); } return result; } diff --git a/tests/fdb/database/test_indexaxis.cc b/tests/fdb/database/test_indexaxis.cc index 190d50d2a..a8e27767d 100644 --- a/tests/fdb/database/test_indexaxis.cc +++ b/tests/fdb/database/test_indexaxis.cc @@ -145,6 +145,33 @@ CASE("Check that merging works correctly") { EXPECT(iatest != ia2); } +CASE("Copy internal map") { + + fdb5::IndexAxis ia; + ia.insert(EXAMPLE_K1); + ia.insert(EXAMPLE_K2); + ia.insert(EXAMPLE_K3); + ia.sort(); + + std::map> map = ia.map(); + + EXPECT(map.size() == 4); + for (const auto& [k, v] : map) { + EXPECT(v == ia.values(k)); + } + + // Make sure it's not a shallow copy + map["class"].insert("new1"); + map["class"].sort(); + EXPECT(map["class"].contains("new1")); + EXPECT(!ia.values("class").contains("new1")); + + ia.insert(fdb5::Key{{{"class", "new2"}}}); + ia.sort(); + EXPECT(ia.values("class").contains("new2")); + EXPECT(!map["class"].contains("new2")); +} + //---------------------------------------------------------------------------------------------------------------------- } // anonymous namespace From ead8f24d75d221c6706407930241f33ae5c1c9a7 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Fri, 5 Apr 2024 08:19:47 +0200 Subject: [PATCH 5/5] fix merge (axis test) --- src/fdb5/database/Key.cc | 13 +++++++++---- tests/fdb/database/test_indexaxis.cc | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/fdb5/database/Key.cc b/src/fdb5/database/Key.cc index fcf2ed7d6..93828217a 100644 --- a/src/fdb5/database/Key.cc +++ b/src/fdb5/database/Key.cc @@ -32,7 +32,9 @@ Key::Key(const std::shared_ptr reg) : Key::Key(const std::string &s, const Rule *rule) : keys_(), - registry_(rule ? rule->registry() : nullptr), canonical_(false) { + registry_(rule ? rule->registry() : nullptr), + canonical_(false) { + eckit::Tokenizer parse(":", true); eckit::StringList values; parse(s, values); @@ -43,7 +45,8 @@ Key::Key(const std::string &s, const Rule *rule) : Key::Key(const eckit::StringDict &keys, const std::shared_ptr reg) : keys_(keys), - registry_(reg), canonical_(false) { + registry_(reg), + canonical_(false) { eckit::StringDict::const_iterator it = keys.begin(); eckit::StringDict::const_iterator end = keys.end(); @@ -53,13 +56,15 @@ Key::Key(const eckit::StringDict &keys, const std::shared_ptr reg } Key::Key(eckit::Stream& s, const std::shared_ptr reg) : - registry_(reg), canonical_(reg==nullptr) { + registry_(reg), + canonical_(reg == nullptr) { decode(s); } Key::Key(std::initializer_list> l, const std::shared_ptr reg) : keys_(l), - registry_(reg), canonical_(false) { + registry_(reg), + canonical_(reg == nullptr) { for (const auto& kv : keys_) { names_.emplace_back(kv.first); diff --git a/tests/fdb/database/test_indexaxis.cc b/tests/fdb/database/test_indexaxis.cc index a8e27767d..de056df6d 100644 --- a/tests/fdb/database/test_indexaxis.cc +++ b/tests/fdb/database/test_indexaxis.cc @@ -18,8 +18,8 @@ fdb5::Key EXAMPLE_K1{{ fdb5::Key EXAMPLE_K2{{ {"class", "rd"}, - {"expver", "0001"}, - {"time", "1200"} + {"expver", "0001"}, + {"time", "1200"} }}; fdb5::Key EXAMPLE_K3{{ @@ -50,9 +50,9 @@ CASE("Insertion and comparison") { EXPECT(ia1 != ia2); ia2.insert(EXAMPLE_K1); - EXPECT(!(ia1 == ia2)); EXPECT(ia1 != ia2); + ia1.sort(); ia2.sort(); EXPECT(ia1 == ia2);