diff --git a/.clang-format b/.clang-format index 053c7ef93..4c528e88a 100644 --- a/.clang-format +++ b/.clang-format @@ -1,108 +1,255 @@ --- -Language: Cpp +Language: Cpp +BasedOnStyle: llvm AccessModifierOffset: -4 AlignAfterOpenBracket: Align -AlignConsecutiveAssignments: true -AlignConsecutiveDeclarations: false +AlignArrayOfStructures: None +AlignConsecutiveAssignments: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: true + AlignFunctionPointers: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveDeclarations: + Enabled: true + AcrossEmptyLines: false + AcrossComments: true + AlignCompound: true + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveMacros: + Enabled: true + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: true + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveShortCaseStatements: + Enabled: true + AcrossEmptyLines: true + AcrossComments: true + AlignCaseColons: false AlignEscapedNewlines: Left -AlignOperands: true -AlignTrailingComments: true +AlignOperands: AlignAfterOperator +AlignTrailingComments: + Kind: Always + OverEmptyLines: 1 +AllowAllArgumentsOnNextLine: false AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: true -AllowShortCaseLabelsOnASingleLine: false +AllowBreakBeforeNoexceptSpecifier: Never +AllowShortBlocksOnASingleLine: Always +AllowShortCaseLabelsOnASingleLine: true +AllowShortCompoundRequirementOnASingleLine: true +AllowShortEnumsOnASingleLine: true AllowShortFunctionsOnASingleLine: Inline -AllowShortIfStatementsOnASingleLine: false -AllowShortLoopsOnASingleLine: false +AllowShortIfStatementsOnASingleLine: AllIfsAndElse +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: true AlwaysBreakAfterDefinitionReturnType: None AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: true +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: Yes +AttributeMacros: + - __capability + - __forceinline + - __host__ + - __noinline + - __nonnull + - __nothrow + - __nullable + - __output + - __unused BinPackArguments: true BinPackParameters: true +BitFieldColonSpacing: Both BraceWrapping: - AfterClass: false - AfterControlStatement: false - AfterEnum: true - AfterFunction: false - AfterNamespace: false + AfterCaseLabel: false + AfterClass: false + AfterControlStatement: Never + AfterEnum: false + AfterExternBlock: false + AfterFunction: false + AfterNamespace: false AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - BeforeCatch: true - BeforeElse: true - IndentBraces: false - SplitEmptyFunction: true - SplitEmptyRecord: true + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: false + SplitEmptyRecord: false SplitEmptyNamespace: true +BreakAdjacentStringLiterals: false +BreakAfterAttributes: Leave +BreakAfterJavaFieldAnnotations: false +BreakArrays: true BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: Always BreakBeforeBraces: Custom -BreakBeforeInheritanceComma: false +BreakBeforeInlineASMColon: OnlyMultiline BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: false -BreakConstructorInitializers: AfterColon -BreakAfterJavaFieldAnnotations: false +BreakConstructorInitializers: BeforeColon +BreakInheritanceList: BeforeColon BreakStringLiterals: true -ColumnLimit: 120 -CommentPragmas: '^ IWYU pragma:' -CompactNamespaces: false -ConstructorInitializerAllOnOneLineOrOnePerLine: true +ColumnLimit: 120 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: true ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 Cpp11BracedListStyle: true DerivePointerAlignment: false -DisableFormat: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock ExperimentalAutoDetectBinPacking: false FixNamespaceComments: true ForEachMacros: - foreach - Q_FOREACH - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve IncludeCategories: - - Regex: '^<.*\.h>' - Priority: 1 - - Regex: '^<.*' - Priority: 2 - - Regex: '.*' - Priority: 3 -IncludeIsMainRegex: '([-_](test|unittest))?$' + - Regex: '^<.*\.h>' + Priority: 2 + - Regex: '^<.*\.hpp>' + Priority: 3 + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 4 + SortPriority: 4 + CaseSensitive: true + - Regex: '^((<|")(gtest|gmock|isl|json)/)' + Priority: 5 + - Regex: '<[[:alnum:]._]+>' + Priority: 6 + - Regex: '.*' + Priority: 1 + SortPriority: 0 +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseBlocks: false IndentCaseLabels: true -IndentWidth: 4 +IndentExternBlock: AfterExternBlock +IndentGotoLabels: true +IndentPPDirectives: None +IndentRequiresClause: true +IndentWidth: 4 IndentWrappedFunctionNames: false +InsertBraces: true +InsertNewlineAtEOF: true +InsertTrailingCommas: None +IntegerLiteralSeparator: + Binary: 0 + BinaryMinDigits: 0 + Decimal: 0 + DecimalMinDigits: 0 + Hex: 0 + HexMinDigits: 0 JavaScriptQuotes: Leave JavaScriptWrapImports: true KeepEmptyLinesAtTheStartOfBlocks: true +KeepEmptyLinesAtEOF: false +LambdaBodyIndentation: Signature +LineEnding: DeriveLF MacroBlockBegin: '' -MacroBlockEnd: '' -MaxEmptyLinesToKeep: 2 +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 NamespaceIndentation: None -ObjCBlockIndentWidth: 2 +ObjCBinPackProtocolList: Never +ObjCBlockIndentWidth: 4 +ObjCBreakBeforeNestedBlockParam: true ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: false -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 1 +ObjCSpaceBeforeProtocolList: true +PackConstructorInitializers: NextLine +PenaltyBreakAssignment: 20 +PenaltyBreakBeforeFirstCallParameter: 19 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakScopeResolution: 500 +PenaltyBreakString: 100000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 10 +PenaltyIndentedWhitespace: 0 +PenaltyReturnTypeOnItsOwnLine: 1000000 PointerAlignment: Left -ReflowComments: true -SortIncludes: true -SortUsingDeclarations: true +PPIndentWidth: -1 +QualifierAlignment: Leave +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +RemoveParentheses: Leave +RemoveSemicolon: false +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope +SeparateDefinitionBlocks: Always +ShortNamespaceLines: 3 +SkipMacroDefinitionBody: false +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: Lexicographic SpaceAfterCStyleCast: false -SpaceAfterTemplateKeyword: true +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: false +SpaceAroundPointerQualifiers: Default SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: true +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeJsonColon: false SpaceBeforeParens: ControlStatements -SpaceInEmptyParentheses: false +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + AfterPlacementOperator: true + AfterRequiresInClause: false + AfterRequiresInExpression: false + BeforeNonEmptyParentheses: false +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: true SpacesBeforeTrailingComments: 2 -SpacesInAngles: false -SpacesInContainerLiterals: true -SpacesInCStyleCastParentheses: false -SpacesInParentheses: false +SpacesInAngles: Never +SpacesInContainerLiterals: false +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParens: Never +SpacesInParensOptions: + InCStyleCasts: false + InConditionalStatements: false + InEmptyParentheses: false + Other: false SpacesInSquareBrackets: false -Standard: Auto -TabWidth: 4 -UseTab: Never -BreakBeforeBinaryOperators: All -ColumnLimit: 0 -... +Standard: c++17 +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 4 +UseTab: Never +VerilogBreakBetweenInstancePorts: true +WhitespaceSensitiveMacros: + - BOOST_PP_STRINGIZE + - CF_SWIFT_NAME + - NS_SWIFT_NAME + - PP_STRINGIZE + - STRINGIZE diff --git a/debug-fdb.sh b/debug-fdb.sh new file mode 100755 index 000000000..4ed63ac90 --- /dev/null +++ b/debug-fdb.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +yell() { echo "$(basename "$0"): $*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export FDB_DIR="~/dev/bundle_stack/fdb" + +export PATH="$FDB_DIR/bin:$PATH" +# export FDB_HOME="$FDB_DIR/build" +export FDB5_CONFIG_FILE="local.yaml" + +REQUEST='class=rd,expver=wxyz,stream=enfo,date=20240902,time=0000,domain=g,type=pf,levtype=ml,step=9,number=9,levelist=9,param=1' + +######################################################################################################################## + +try time fdb-inspect $REQUEST + +try xctrace record --template 'Metins Instrument' --launch -- $FDB_DIR/bin/fdb-inspect $REQUEST diff --git a/fdb-write-test-db.sh b/fdb-write-test-db.sh new file mode 100755 index 000000000..495f49393 --- /dev/null +++ b/fdb-write-test-db.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +yell() { echo "$(basename "$0"): $*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +export FDB_DIR=~/dev/bundle_stack/fdb + +# export PATH=~/workspace/bundle_stack/build/bin:$PATH +# export ECCODES_DEFINITION_PATH=~/workspace/bundle_stack/build/share/eccodes/definitions +# export FDB_HOME="$FDB_DIR/build" +export FDB5_CONFIG_FILE="local.yaml" + +DB_NAME=large_fdb_test +BASE_GRIB=test.grib + +######################################################################################################################## + +# rm -rf "./$DB_NAME" || true +# mkdir -p "./$DB_NAME/localroot" + +try cd ./$DB_NAME + +try cp "$FDB_DIR/$BASE_GRIB" . +try cp "$FDB_DIR/build/etc/fdb/schema" . +try cp "$FDB_DIR/tests/fdb/tools/list/local.yaml" . + +######################################################################################################################## + +echo "Populating testing FDB using $BASE_GRIB" + +for i in $(seq -f "%02g" 1 2); do + # echo "number $i" + try grib_set -s date=202409"$i" $BASE_GRIB tmp.grib + + # fdb-hammer --class=od --expver=0001 --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib + # fdb-hammer --class=od --expver=xxxx --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib + # fdb-hammer --class=od --expver=wxyz --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib + + fdb-hammer --class=rd --expver=0001 --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib + fdb-hammer --class=rd --expver=xxxx --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib + fdb-hammer --class=rd --expver=wxyz --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib + +# fdb-hammer --class=ea --expver=0001 --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib +# fdb-hammer --class=ea --expver=xxxx --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib +# fdb-hammer --class=ea --expver=wxyz --nlevels=5 --nparams=5 --nensembles=10 --nsteps=10 tmp.grib + +done diff --git a/fdb_test_env.sh b/fdb_test_env.sh new file mode 100644 index 000000000..a71b84c71 --- /dev/null +++ b/fdb_test_env.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +export PATH="$HOME/dev/bundle_stack/eccodes/build/bin/:$HOME/dev/bundle_stack/fdb/build/bin:$PATH" + +export FDB_HOME="$HOME/dev/bundle_stack/fdb/build" + +export FDB5_CONFIG_FILE="local.yaml" diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 41f61c69b..e4074d1e2 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -39,7 +39,8 @@ list( APPEND fdb5_srcs api/helpers/FDBToolRequest.cc api/helpers/FDBToolRequest.h api/helpers/DumpIterator.h - api/helpers/ListIterator.cc + api/helpers/ListElement.cc + api/helpers/ListElement.h api/helpers/ListIterator.h api/helpers/LockIterator.h api/helpers/MoveIterator.h @@ -126,6 +127,7 @@ list( APPEND fdb5_srcs database/IndexFactory.h database/Key.cc database/Key.h + database/KeyChain.h database/ReadVisitor.cc database/ReadVisitor.h database/Report.cc diff --git a/src/fdb5/api/DistFDB.cc b/src/fdb5/api/DistFDB.cc index fbbe524b5..37b28a247 100644 --- a/src/fdb5/api/DistFDB.cc +++ b/src/fdb5/api/DistFDB.cc @@ -179,11 +179,12 @@ auto DistFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) -> } -ListIterator DistFDB::list(const FDBToolRequest& request) { +ListIterator DistFDB::list(const FDBToolRequest& request, int level) { LOG_DEBUG_LIB(LibFdb5) << "DistFDB::list() : " << request << std::endl; return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.list(request); + [level](FDB& fdb, const FDBToolRequest& request) { + bool deduplicate = false; // never deduplicate on inner calls + return fdb.list(request, deduplicate, level); }); } diff --git a/src/fdb5/api/DistFDB.h b/src/fdb5/api/DistFDB.h index 494009e22..cbd8f3048 100644 --- a/src/fdb5/api/DistFDB.h +++ b/src/fdb5/api/DistFDB.h @@ -45,7 +45,7 @@ class DistFDB : public FDBBase { ListIterator inspect(const metkit::mars::MarsRequest& request) override; - ListIterator list(const FDBToolRequest& request) override; + ListIterator list(const FDBToolRequest& request, int level) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override; diff --git a/src/fdb5/api/FDB.cc b/src/fdb5/api/FDB.cc index 573d2afc2..3940e8f8a 100644 --- a/src/fdb5/api/FDB.cc +++ b/src/fdb5/api/FDB.cc @@ -13,10 +13,17 @@ * (Project ID: 671951) www.nextgenio.eu */ +#include +#include +#include +#include + #include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" #include "eckit/io/DataHandle.h" #include "eckit/io/MemoryHandle.h" #include "eckit/log/Log.h" +#include "eckit/log/Timer.h" #include "eckit/message/Message.h" #include "eckit/message/Reader.h" @@ -26,6 +33,9 @@ #include "fdb5/api/FDB.h" #include "fdb5/api/FDBFactory.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListElement.h" +#include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/database/FieldLocation.h" #include "fdb5/database/Key.h" #include "fdb5/io/HandleGatherer.h" #include "fdb5/message/MessageDecoder.h" @@ -180,7 +190,7 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { if (it.next(el)) { // build the request representing the tensor-product of all retrieved fields metkit::mars::MarsRequest cubeRequest = el.combinedKey().request(); - std::vector elements{el}; + std::vector elements {el}; while (it.next(el)) { cubeRequest.merge(el.combinedKey().request()); @@ -188,11 +198,9 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { } // checking all retrieved fields against the hypercube, to remove duplicates - ListElementDeduplicator dedup; - metkit::hypercube::HyperCubePayloaded cube(cubeRequest, dedup); - for(auto el: elements) { - cube.add(el.combinedKey().request(), el); - } + ListElementDeduplicator deduplicator; + metkit::hypercube::HyperCubePayloaded cube(cubeRequest, deduplicator); + for (const auto& elem : elements) { cube.add(elem.combinedKey().request(), el); } if (cube.countVacant() > 0) { std::stringstream ss; @@ -203,7 +211,7 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) { eckit::Log::warning() << ss.str() << std::endl; } - for (size_t i=0; i< cube.size(); i++) { + for (std::size_t i = 0; i < cube.size(); i++) { ListElement element; if (cube.find(i, element)) { result.add(element.location().dataHandle()); @@ -228,8 +236,8 @@ ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) { return internal_->inspect(request); } -ListIterator FDB::list(const FDBToolRequest& request, bool deduplicate) { - return ListIterator(internal_->list(request), deduplicate); +ListIterator FDB::list(const FDBToolRequest& request, const bool deduplicate, const int level) { + return {internal_->list(request, level), deduplicate}; } DumpIterator FDB::dump(const FDBToolRequest& request, bool simple) { diff --git a/src/fdb5/api/FDB.h b/src/fdb5/api/FDB.h index 6be852565..f27c1e634 100644 --- a/src/fdb5/api/FDB.h +++ b/src/fdb5/api/FDB.h @@ -21,6 +21,7 @@ #include #include +#include #include "eckit/distributed/Transport.h" @@ -94,7 +95,7 @@ class FDB { ListIterator inspect(const metkit::mars::MarsRequest& request); - ListIterator list(const FDBToolRequest& request, bool deduplicate=false); + ListIterator list(const FDBToolRequest& request, bool deduplicate=false, int level=3); DumpIterator dump(const FDBToolRequest& request, bool simple=false); diff --git a/src/fdb5/api/FDBFactory.h b/src/fdb5/api/FDBFactory.h index 6cd3c1c37..9d28e35b9 100644 --- a/src/fdb5/api/FDBFactory.h +++ b/src/fdb5/api/FDBFactory.h @@ -71,7 +71,7 @@ class FDBBase : private eckit::NonCopyable { virtual ListIterator inspect(const metkit::mars::MarsRequest& request) = 0; - virtual ListIterator list(const FDBToolRequest& request) = 0; + virtual ListIterator list(const FDBToolRequest& request, int level) = 0; virtual DumpIterator dump(const FDBToolRequest& request, bool simple) = 0; diff --git a/src/fdb5/api/LocalFDB.cc b/src/fdb5/api/LocalFDB.cc index 3d86255e4..8de5ef1f0 100644 --- a/src/fdb5/api/LocalFDB.cc +++ b/src/fdb5/api/LocalFDB.cc @@ -15,7 +15,6 @@ #include "eckit/container/Queue.h" #include "eckit/log/Log.h" -#include "eckit/message/Message.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/api/helpers/FDBToolRequest.h" @@ -34,7 +33,6 @@ #include "fdb5/api/local/DumpVisitor.h" #include "fdb5/api/local/ListVisitor.h" #include "fdb5/api/local/PurgeVisitor.h" -#include "fdb5/api/local/QueryVisitor.h" #include "fdb5/api/local/StatsVisitor.h" #include "fdb5/api/local/StatusVisitor.h" #include "fdb5/api/local/WipeVisitor.h" @@ -83,9 +81,9 @@ APIIterator LocalFDB::queryInternal(const FDBTo return QueryIterator(new AsyncIterator(async_worker)); } -ListIterator LocalFDB::list(const FDBToolRequest& request) { +ListIterator LocalFDB::list(const FDBToolRequest& request, const int level) { LOG_DEBUG_LIB(LibFdb5) << "LocalFDB::list() : " << request << std::endl; - return queryInternal(request); + return queryInternal(request, level); } DumpIterator LocalFDB::dump(const FDBToolRequest &request, bool simple) { diff --git a/src/fdb5/api/LocalFDB.h b/src/fdb5/api/LocalFDB.h index 96f4a6914..bb654640b 100644 --- a/src/fdb5/api/LocalFDB.h +++ b/src/fdb5/api/LocalFDB.h @@ -41,7 +41,7 @@ class LocalFDB : public FDBBase { ListIterator inspect(const metkit::mars::MarsRequest& request) override; - ListIterator list(const FDBToolRequest& request) override; + ListIterator list(const FDBToolRequest& request, int level) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override; diff --git a/src/fdb5/api/RemoteFDB.cc b/src/fdb5/api/RemoteFDB.cc index c5e0b74e9..044d222f7 100644 --- a/src/fdb5/api/RemoteFDB.cc +++ b/src/fdb5/api/RemoteFDB.cc @@ -13,28 +13,34 @@ * (Project ID: 671951) www.nextgenio.eu */ -#include #include +#include +#include +#include +#include +#include #include "fdb5/api/RemoteFDB.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/io/HandleGatherer.h" #include "fdb5/remote/Messages.h" #include "fdb5/remote/RemoteFieldLocation.h" #include "fdb5/api/helpers/FDBToolRequest.h" #include "fdb5/database/Key.h" +#include "eckit/config/Configuration.h" #include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/exception/Exceptions.h" #include "eckit/io/Buffer.h" #include "eckit/log/Bytes.h" #include "eckit/log/Log.h" #include "eckit/message/Message.h" -#include "eckit/distributed/Transport.h" -#include "eckit/config/Resource.h" +#include "eckit/os/BackTrace.h" +#include "eckit/runtime/Main.h" #include "eckit/serialisation/MemoryStream.h" #include "eckit/utils/Translator.h" -#include "eckit/runtime/Main.h" -#include "eckit/os/BackTrace.h" #include "metkit/mars/MarsRequest.h" @@ -62,7 +68,7 @@ class ConnectionError : public eckit::Exception { ConnectionError(const int); ConnectionError(const int, const eckit::net::Endpoint&); - bool retryOnClient() const override { return true; } + bool retryOnClient() const override { return true; } }; ConnectionError::ConnectionError(const int retries) { @@ -549,13 +555,20 @@ struct BaseAPIHelper { static ValueType valueFromStream(eckit::Stream& s, RemoteFDB* fdb) { return ValueType(s); } }; -using ListHelper = BaseAPIHelper; +struct ListHelper : public BaseAPIHelper { + + ListHelper(int level) : level_(level) {} + void encodeExtra(eckit::Stream& s) const { s << level_; } + +private: + int level_; +}; struct InspectHelper : BaseAPIHelper { static ListElement valueFromStream(eckit::Stream& s, RemoteFDB* fdb) { ListElement elem(s); - return ListElement(elem.key(), RemoteFieldLocation(fdb, elem.location()).make_shared(), elem.timestamp()); + return {elem.keys(), std::make_shared(fdb, elem.location()), elem.timestamp()}; } }; @@ -625,7 +638,7 @@ struct WipeHelper : BaseAPIHelper { struct MoveHelper : BaseAPIHelper { - + MoveHelper(const eckit::URI& dest) : dest_(dest) {} @@ -715,8 +728,8 @@ auto RemoteFDB::forwardApiCall(const HelperClass& helper, const FDBToolRequest& ); } -ListIterator RemoteFDB::list(const FDBToolRequest& request) { - return forwardApiCall(ListHelper(), request); +ListIterator RemoteFDB::list(const FDBToolRequest& request, const int level) { + return forwardApiCall(ListHelper(level), request); } ListIterator RemoteFDB::inspect(const metkit::mars::MarsRequest& request) { diff --git a/src/fdb5/api/RemoteFDB.h b/src/fdb5/api/RemoteFDB.h index 38f83e4f2..084dd57e4 100644 --- a/src/fdb5/api/RemoteFDB.h +++ b/src/fdb5/api/RemoteFDB.h @@ -62,7 +62,7 @@ class RemoteFDB : public FDBBase { ListIterator inspect(const metkit::mars::MarsRequest& request) override; - ListIterator list(const FDBToolRequest& request) override; + ListIterator list(const FDBToolRequest& request, int level) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override; diff --git a/src/fdb5/api/SelectFDB.cc b/src/fdb5/api/SelectFDB.cc index 586ee5b19..477255f8f 100644 --- a/src/fdb5/api/SelectFDB.cc +++ b/src/fdb5/api/SelectFDB.cc @@ -137,12 +137,10 @@ auto SelectFDB::queryInternal(const FDBToolRequest& request, const QueryFN& fn) return QueryIterator(new APIAggregateIterator(std::move(iterQueue))); } -ListIterator SelectFDB::list(const FDBToolRequest& request) { +ListIterator SelectFDB::list(const FDBToolRequest& request, const int level) { LOG_DEBUG_LIB(LibFdb5) << "SelectFDB::list() >> " << request << std::endl; return queryInternal(request, - [](FDB& fdb, const FDBToolRequest& request) { - return fdb.list(request); - }); + [level](FDB& fdb, const FDBToolRequest& request) { return fdb.list(request, false, level); }); } DumpIterator SelectFDB::dump(const FDBToolRequest& request, bool simple) { diff --git a/src/fdb5/api/SelectFDB.h b/src/fdb5/api/SelectFDB.h index 78937c358..4407f7218 100644 --- a/src/fdb5/api/SelectFDB.h +++ b/src/fdb5/api/SelectFDB.h @@ -50,7 +50,7 @@ class SelectFDB : public FDBBase { ListIterator inspect(const metkit::mars::MarsRequest& request) override; - ListIterator list(const FDBToolRequest& request) override; + ListIterator list(const FDBToolRequest& request, int level) override; DumpIterator dump(const FDBToolRequest& request, bool simple) override; diff --git a/src/fdb5/api/fdb_c.cc b/src/fdb5/api/fdb_c.cc index 79aa7fc84..dc075fcb8 100644 --- a/src/fdb5/api/fdb_c.cc +++ b/src/fdb5/api/fdb_c.cc @@ -8,20 +8,22 @@ * does it submit to any jurisdiction. */ +#include "eckit/exception/Exceptions.h" #include "eckit/io/MemoryHandle.h" -#include "eckit/io/FileDescHandle.h" #include "eckit/message/Message.h" #include "eckit/runtime/Main.h" +#include "eckit/utils/Tokenizer.h" #include "metkit/mars/MarsRequest.h" #include "metkit/mars/MarsExpension.h" -#include "eckit/utils/Tokenizer.h" -#include "fdb5/fdb5_version.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/api/helpers/ListIterator.h" #include "fdb5/database/Key.h" +#include "fdb5/database/KeyChain.h" +#include "fdb5/fdb5_version.h" #include "fdb5/api/fdb_c.h" @@ -88,50 +90,51 @@ struct fdb_request_t { }; struct fdb_split_key_t { -public: - fdb_split_key_t() : key_(nullptr), level_(-1) {} + using value_type = KeyChain; - void set(const std::vector& key) { - key_ = &key; - level_ = -1; + auto operator=(const value_type& keys) -> fdb_split_key_t& { + keys_ = &keys; + level_ = keys_->end(); + return *this; } - int next_metadata(const char** k, const char** v, size_t* level) { - if (key_ == nullptr) { - std::stringstream ss; - ss << "fdb_split_key_t not valid. Key not configured"; - throw eckit::UserError(ss.str(), Here()); + auto operator++() -> fdb_split_key_t& { + /// @todo the following "if" is an unfortunate consequence of a flaw in this iterator + if (level_ == keys_->end()) { + level_ = keys_->begin(); + curr_ = level_->begin(); + return *this; } - if (level_ == -1) { - if (0 < key_->size()) { - level_ = 0; - it_ = key_->at(0).begin(); - } else { - return FDB_ITERATION_COMPLETE; - } - } - while (it_ == key_->at(level_).end()) { - if (level_size()-1) { - level_++; - it_ = key_->at(level_).begin(); - } else { - return FDB_ITERATION_COMPLETE; - } + if (curr_ != level_->end()) { + ++curr_; + if (curr_ == level_->end() && level_ != keys_->end() - 1) { curr_ = (++level_)->begin(); } } + return *this; + } - *k = it_->first.c_str(); - *v = it_->second.c_str(); - if (level != nullptr) { - *level = level_; - } - it_++; + int next() { + ++(*this); + if (curr_ == level_->end()) { return FDB_ITERATION_COMPLETE; } return FDB_SUCCESS; } -private: - const std::vector* key_; - int level_; - Key::const_iterator it_; + void metadata(const char** k, const char** v, size_t* level) const { + ASSERT_MSG(keys_, "keys are missing!"); + + const auto& [key, val] = *curr_; + + *k = key.c_str(); + *v = val.c_str(); + + if (level) { *level = level_ - keys_->begin(); } + } + +private: // members + const value_type* keys_ {nullptr}; + + value_type::const_iterator level_; + + Key::const_iterator curr_; }; struct fdb_listiterator_t { @@ -147,17 +150,20 @@ struct fdb_listiterator_t { void attrs(const char** uri, size_t* off, size_t* len) { ASSERT(validEl_); - const FieldLocation& loc = el_.location(); - *uri = loc.uri().name().c_str(); - *off = loc.offset(); - *len = loc.length(); + // guard against negative values + ASSERT(0 <= el_.offset()); + ASSERT(0 <= el_.length()); + + *uri = el_.uri().name().c_str(); + *off = el_.offset(); + *len = el_.length(); } void key(fdb_split_key_t* key) { ASSERT(validEl_); ASSERT(key); - key->set(el_.key()); + *key = el_.keys(); } private: @@ -197,7 +203,7 @@ struct fdb_datareader_t { delete dh_; dh_ = dh; } - + private: DataHandle* dh_; }; @@ -355,17 +361,18 @@ int fdb_archive_multiple(fdb_handle_t* fdb, fdb_request_t* req, const char* data }); } -int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates) { - return wrapApiFunction([fdb, req, it, duplicates] { +int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, const bool duplicates, const int depth) { + return wrapApiFunction([fdb, req, it, duplicates, depth] { ASSERT(fdb); ASSERT(it); + ASSERT(depth >= 1 && depth <= 3); std::vector minKeySet; // we consider an empty set const FDBToolRequest toolRequest( req ? req->request() : metkit::mars::MarsRequest(), req == nullptr, minKeySet); - *it = new fdb_listiterator_t(fdb->list(toolRequest, duplicates)); + *it = new fdb_listiterator_t(fdb->list(toolRequest, duplicates, depth)); }); } int fdb_retrieve(fdb_handle_t* fdb, fdb_request_t* req, fdb_datareader_t* dr) { @@ -486,7 +493,9 @@ int fdb_splitkey_next_metadata(fdb_split_key_t* it, const char** key, const char ASSERT(it); ASSERT(key); ASSERT(value); - return it->next_metadata(key, value, level); + const auto stat = it->next(); + it->metadata(key, value, level); + return stat; }}); } int fdb_delete_splitkey(fdb_split_key_t* key) { diff --git a/src/fdb5/api/fdb_c.h b/src/fdb5/api/fdb_c.h index e3371c61a..8cd2ae5e5 100644 --- a/src/fdb5/api/fdb_c.h +++ b/src/fdb5/api/fdb_c.h @@ -50,7 +50,7 @@ int fdb_version(const char** version); * \param version Return variable for version control checksum. Returned pointer valid throughout program lifetime. * \returns Return code (#FdbErrorValues) */ -int fdb_vcs_version(const char** version); +int fdb_vcs_version(const char** sha1); ///@} @@ -336,7 +336,7 @@ int fdb_archive_multiple(fdb_handle_t* fdb, fdb_request_t* req, const char* data * \param duplicates Boolean flag used to specify if duplicated ListElements are to be reported or not. * \returns Return code (#FdbErrorValues) */ -int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates); +int fdb_list(fdb_handle_t* fdb, const fdb_request_t* req, fdb_listiterator_t** it, bool duplicates, int depth); /** Return all available data whose metadata matches a given user request. * \param fdb FDB instance. diff --git a/src/fdb5/api/helpers/ListElement.cc b/src/fdb5/api/helpers/ListElement.cc new file mode 100644 index 000000000..ce4fd6d4d --- /dev/null +++ b/src/fdb5/api/helpers/ListElement.cc @@ -0,0 +1,117 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#include "fdb5/api/helpers/ListElement.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/log/JSON.h" +#include "eckit/serialisation/Reanimator.h" +#include "eckit/serialisation/Stream.h" +#include "fdb5/database/FieldLocation.h" +#include "fdb5/database/Key.h" +#include "fdb5/database/KeyChain.h" + +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +ListElement::ListElement(Key dbKey, const eckit::URI& uri, const TimeStamp& timestamp): + keys_ {std::move(dbKey)}, uri_ {uri}, timestamp_ {timestamp} { } + +ListElement::ListElement(Key dbKey, Key indexKey, const eckit::URI& uri, const TimeStamp& timestamp): + keys_ {std::move(dbKey), std::move(indexKey)}, uri_ {uri}, timestamp_ {timestamp} { } + +ListElement::ListElement(Key dbKey, Key indexKey, Key datumKey, std::shared_ptr location, + const TimeStamp& timestamp): + keys_ {std::move(dbKey), std::move(indexKey), std::move(datumKey)}, + loc_ {std::move(location)}, + timestamp_ {timestamp} { } + +ListElement::ListElement(const KeyChain& keys, std::shared_ptr location, const TimeStamp& timestamp): + ListElement(keys[0], keys[1], keys[2], std::move(location), timestamp) { } + +ListElement::ListElement(eckit::Stream& stream) { + stream >> keys_; + stream >> uri_; + loc_.reset(eckit::Reanimator::reanimate(stream)); + stream >> timestamp_; +} + +const FieldLocation& ListElement::location() const { + if (!loc_) { throw eckit::SeriousBug("Only datum (3-level) elements have FieldLocation.", Here()); } + return *loc_; +} + +const eckit::URI& ListElement::uri() const { + if (loc_) { return loc_->uri(); } + return uri_; +} + +eckit::Offset ListElement::offset() const { + return loc_ ? loc_->offset() : eckit::Offset(0); +} + +eckit::Length ListElement::length() const { + return loc_ ? loc_->length() : eckit::Length(0); +} + +void ListElement::print(std::ostream& out, const bool location, const bool length, const bool timestamp, const char* sep) const { + out << keys_; + if (location) { + out << sep; + if (loc_) { + out << *loc_; + } else { + out << "host=" << (uri_.host().empty() ? "empty" : uri_.host()); + } + } + if (length) { out << sep << "length=" << this->length(); } + if (timestamp) { out << sep << "timestamp=" << timestamp_; } +} + +void ListElement::json(eckit::JSON& json) const { + json << keys_.combine().keyDict(); + if (loc_) { json << "length" << loc_->length(); } +} + +void ListElement::encode(eckit::Stream& stream) const { + for (const auto& key : keys_) { stream << key; } + stream << uri_; + if (loc_) { stream << *loc_; } + stream << timestamp_; +} + +//---------------------------------------------------------------------------------------------------------------------- +// friends + +std::ostream& operator<<(std::ostream& out, const ListElement& elem) { + elem.print(out, false, false, false, " "); + return out; +} + +eckit::Stream& operator<<(eckit::Stream& stream, const ListElement& elem) { + elem.encode(stream); + return stream; +} + +eckit::JSON& operator<<(eckit::JSON& json, const ListElement& elem) { + elem.json(json); + return json; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/api/helpers/ListElement.h b/src/fdb5/api/helpers/ListElement.h new file mode 100644 index 000000000..625868d6b --- /dev/null +++ b/src/fdb5/api/helpers/ListElement.h @@ -0,0 +1,101 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Simon Smart +/// @author Emanuele Danovaro +/// @author Metin Cakircali +/// @date October 2018 + +#pragma once + +#include "eckit/filesystem/URI.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" +#include "fdb5/database/Key.h" +#include "fdb5/database/KeyChain.h" + +#include +#include +#include + +namespace eckit { +class JSON; +class Stream; +} + +namespace fdb5 { + +class FieldLocation; + +//---------------------------------------------------------------------------------------------------------------------- + +/// Define a standard object which can be used to iterate the results of a +/// list() call on an arbitrary FDB object + +class ListElement { +public: // types + using TimeStamp = std::time_t; + +public: // methods + ListElement(Key dbKey, const eckit::URI& uri, const TimeStamp& timestamp); + + ListElement(Key dbKey, Key indexKey, const eckit::URI& uri, const TimeStamp& timestamp); + + ListElement(Key dbKey, Key indexKey, Key datumKey, std::shared_ptr location, + const TimeStamp& timestamp); + + ListElement(const KeyChain& keys, std::shared_ptr location, const TimeStamp& timestamp); + + explicit ListElement(eckit::Stream& stream); + + ListElement() = default; + + const KeyChain& keys() const { return keys_; } + + const Key& combinedKey() const { return keys_.combine(); } + + const eckit::URI& uri() const; + + const TimeStamp& timestamp() const { return timestamp_; } + + std::shared_ptr sharedLocation() const { return loc_; } + + const FieldLocation& location() const; + + eckit::Offset offset() const; + + eckit::Length length() const; + + void print(std::ostream& out, bool location, bool length, bool timestamp, const char* sep) const; + +private: // methods + void encode(eckit::Stream& stream) const; + + void json(eckit::JSON& json) const; + + friend std::ostream& operator<<(std::ostream& out, const ListElement& elem); + + friend eckit::Stream& operator<<(eckit::Stream& stream, const ListElement& elem); + + friend eckit::JSON& operator<<(eckit::JSON& json, const ListElement& elem); + +private: // members + KeyChain keys_; + + eckit::URI uri_; + + std::shared_ptr loc_; + + TimeStamp timestamp_ {0}; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/api/helpers/ListIterator.cc b/src/fdb5/api/helpers/ListIterator.cc deleted file mode 100644 index 811013899..000000000 --- a/src/fdb5/api/helpers/ListIterator.cc +++ /dev/null @@ -1,69 +0,0 @@ -/* - * (C) Copyright 1996- ECMWF. - * - * This software is licensed under the terms of the Apache Licence Version 2.0 - * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - * In applying this licence, ECMWF does not waive the privileges and immunities - * granted to it by virtue of its status as an intergovernmental organisation nor - * does it submit to any jurisdiction. - */ - -/* - * This software was developed as part of the EC H2020 funded project NextGenIO - * (Project ID: 671951) www.nextgenio.eu - */ - -#include "fdb5/api/helpers/ListIterator.h" - -#include "eckit/log/JSON.h" - -namespace fdb5 { - -//---------------------------------------------------------------------------------------------------------------------- - -ListElement::ListElement(const std::vector& keyParts, std::shared_ptr location, time_t timestamp) : - keyParts_(keyParts), location_(location), timestamp_(timestamp) {} - -ListElement::ListElement(eckit::Stream &s) { - s >> keyParts_; - location_.reset(eckit::Reanimator::reanimate(s)); - s >> timestamp_; -} - -Key ListElement::combinedKey() const { - Key combined = keyParts_[2]; - - for (const Key& partKey : keyParts_) { - for (const auto& kv : partKey) { - combined.set(kv.first, kv.second); - } - } - return combined; -} - -void ListElement::print(std::ostream &out, bool withLocation, bool withLength, bool withTimestamp, const char* sep) const { - if (!withLocation && location_ && !location_->host().empty()) { - out << "host=" << location_->host() << ","; - } - for (const auto& bit : keyParts_) { - out << bit; - } - if (location_ && withLocation) out << sep << *location_; - if (withLength) out << sep << "length=" << location_->length(); - if (withTimestamp) out << sep << "timestamp=" << timestamp_; -} - -void ListElement::json(eckit::JSON& json) const { - json << combinedKey().keyDict(); - json << "length" << location_->length(); -} - -void ListElement::encode(eckit::Stream &s) const { - s << keyParts_; - s << *location_; - s << timestamp_; -} - -//---------------------------------------------------------------------------------------------------------------------- - -} // namespace fdb5 diff --git a/src/fdb5/api/helpers/ListIterator.h b/src/fdb5/api/helpers/ListIterator.h index 79788d0ef..341afeba9 100644 --- a/src/fdb5/api/helpers/ListIterator.h +++ b/src/fdb5/api/helpers/ListIterator.h @@ -19,75 +19,17 @@ #ifndef fdb5_ListIterator_H #define fdb5_ListIterator_H -#include #include -#include -#include -#include +#include #include "fdb5/database/Key.h" -#include "fdb5/database/FieldLocation.h" #include "fdb5/api/helpers/APIIterator.h" - -namespace eckit { - class Stream; - class JSON; -} +#include "fdb5/api/helpers/ListElement.h" namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- -/// Define a standard object which can be used to iterate the results of a -/// list() call on an arbitrary FDB object - -class ListElement { -public: // methods - - ListElement() = default; - ListElement(const std::vector& keyParts, std::shared_ptr location, time_t timestamp); - ListElement(eckit::Stream& s); - - const std::vector& key() const { return keyParts_; } - const FieldLocation& location() const { return *location_; } - const time_t& timestamp() const { return timestamp_; } - - Key combinedKey() const; - - void print(std::ostream& out, bool withLocation=false, bool withLength=false, bool withTimestamp=false, const char* sep = " ") const; - void json(eckit::JSON& json) const; - -private: // methods - - void encode(eckit::Stream& s) const; - - friend std::ostream& operator<<(std::ostream& os, const ListElement& e) { - e.print(os); - return os; - } - - friend eckit::Stream& operator<<(eckit::Stream& s, const ListElement& r) { - r.encode(s); - return s; - } - - friend eckit::JSON& operator<<(eckit::JSON& j, const ListElement& e) { - e.json(j); - return j; - } - -public: // members - - std::vector keyParts_; - -private: // members - - std::shared_ptr location_; - time_t timestamp_; -}; - -//---------------------------------------------------------------------------------------------------------------------- - using ListAggregateIterator = APIAggregateIterator; using ListAsyncIterator = APIAsyncIterator; @@ -113,16 +55,10 @@ class ListIterator : public APIIterator { ListElement tmp; while (APIIterator::next(tmp)) { if(deduplicate_) { - Key combinedKey = tmp.combinedKey(); - if (seenKeys_.find(combinedKey) == seenKeys_.end()) { - seenKeys_.emplace(std::move(combinedKey)); - std::swap(elem, tmp); - return true; - } - } else { - std::swap(elem, tmp); - return true; + if (const auto& [iter, success] = seenKeys_.insert(tmp.combinedKey()); !success) { continue; } } + std::swap(elem, tmp); + return true; } return false; } diff --git a/src/fdb5/api/local/ListVisitor.h b/src/fdb5/api/local/ListVisitor.h index 938eeec5e..32ad23f8b 100644 --- a/src/fdb5/api/local/ListVisitor.h +++ b/src/fdb5/api/local/ListVisitor.h @@ -19,14 +19,21 @@ #ifndef fdb5_api_local_ListVisitor_H #define fdb5_api_local_ListVisitor_H -#include "fdb5/database/DB.h" -#include "fdb5/database/Index.h" +#include "eckit/container/Queue.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "fdb5/api/helpers/ControlIterator.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/api/local/QueryVisitor.h" -#include "fdb5/api/helpers/ListIterator.h" +#include "fdb5/database/Catalogue.h" +#include "fdb5/database/EntryVisitMechanism.h" +#include "fdb5/database/Field.h" +#include "fdb5/database/Index.h" +#include "metkit/mars/MarsRequest.h" + +#include -namespace fdb5 { -namespace api { -namespace local { +namespace fdb5::api::local { /// @note Helper classes for LocalFDB @@ -35,7 +42,8 @@ namespace local { struct ListVisitor : public QueryVisitor { public: - using QueryVisitor::QueryVisitor; + ListVisitor(eckit::Queue& queue, const metkit::mars::MarsRequest& request, int level): + QueryVisitor(queue, request), level_(level) { } /// Make a note of the current database. Subtract its key from the current /// request so we can test request is used in its entirety @@ -55,6 +63,11 @@ struct ListVisitor : public QueryVisitor { indexRequest_.unsetValues(kv.first); } + if (level_ == 1) { + queue_.emplace(currentCatalogue_->key(), eckit::URI {}, 0); + ret = false; + } + return ret; } @@ -73,9 +86,14 @@ struct ListVisitor : public QueryVisitor { } if (index.partialMatch(request_)) { - return true; // Explore contained entries + if (level_ == 2) { + queue_.emplace(currentCatalogue_->key(), currentIndex_->key(), eckit::URI {}, 0); + return false; + } + return true; // Explore contained entries } - return false; // Skip contained entries + + return false; // Skip contained entries } /// Test if entry matches the current request. If so, add to the output queue. @@ -84,7 +102,7 @@ struct ListVisitor : public QueryVisitor { ASSERT(currentIndex_); if (key.match(datumRequest_)) { - queue_.emplace(ListElement({currentCatalogue_->key(), currentIndex_->key(), key}, field.stableLocation(), field.timestamp())); + queue_.emplace(currentCatalogue_->key(), currentIndex_->key(), key, field.stableLocation(), field.timestamp()); } } @@ -96,12 +114,11 @@ struct ListVisitor : public QueryVisitor { metkit::mars::MarsRequest indexRequest_; metkit::mars::MarsRequest datumRequest_; + const int level_; }; //---------------------------------------------------------------------------------------------------------------------- -} // namespace local -} // namespace api -} // namespace fdb5 +} // namespace fdb5::api::local #endif diff --git a/src/fdb5/database/FieldLocation.h b/src/fdb5/database/FieldLocation.h index e90d2496d..946592141 100644 --- a/src/fdb5/database/FieldLocation.h +++ b/src/fdb5/database/FieldLocation.h @@ -36,8 +36,7 @@ namespace fdb5 { class FieldLocationVisitor; -class FieldLocation : public eckit::OwnedLock, public eckit::Streamable { - +class FieldLocation: public eckit::OwnedLock, public eckit::Streamable { public: // methods FieldLocation() : offset_(eckit::Offset(0)), length_(eckit::Length(0)), remapKey_(Key()) {} diff --git a/src/fdb5/database/KeyChain.h b/src/fdb5/database/KeyChain.h new file mode 100644 index 000000000..c11ab734c --- /dev/null +++ b/src/fdb5/database/KeyChain.h @@ -0,0 +1,81 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file KeyChain.h +/// @author Metin Cakircali +/// @date Aug 2024 + +#pragma once + +#include "eckit/serialisation/Stream.h" +#include "fdb5/database/Key.h" + +#include +#include +#include +#include + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- +/// @brief KeyChain is a set of 3-level keys, where each level is a Key object. +/// Level 1 is the database key, level 2 is the index key, and level 3 is the datum key. + +class KeyChain: public std::array { +public: // methods + KeyChain() = default; + + KeyChain(Key&& dbKey, Key&& indexKey, Key&& datumKey): + std::array {std::move(dbKey), std::move(indexKey), std::move(datumKey)} { } + + KeyChain(Key&& dbKey, Key&& indexKey): KeyChain(std::move(dbKey), std::move(indexKey), Key {}) { } + + explicit KeyChain(Key&& dbKey): KeyChain(std::move(dbKey), Key {}, Key {}) { } + + auto combine() const -> const Key& { + if (!key_) { + /// @todo fixme: no API to get registry from Key; fix this copy hack + key_.emplace((*this)[0]); + for (auto&& key : *this) { + for (auto&& [keyword, value] : key) { key_->set(keyword, value); } + } + } + return *key_; + } + +private: // methods + void print(std::ostream& out) const { + for (auto&& key : *this) { + if (!key.empty()) { out << key; } + } + } + + friend std::ostream& operator<<(std::ostream& out, const KeyChain& keys) { + keys.print(out); + return out; + } + + friend eckit::Stream& operator<<(eckit::Stream& out, const KeyChain& keys) { + for (auto&& key : keys) { out << key; } + return out; + } + + friend eckit::Stream& operator>>(eckit::Stream& out, KeyChain& keys) { + for (auto&& key : keys) { out >> key; } + return out; + } + +private: // members + mutable std::optional key_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/database/Manager.cc b/src/fdb5/database/Manager.cc index a9b033f19..3774268ae 100644 --- a/src/fdb5/database/Manager.cc +++ b/src/fdb5/database/Manager.cc @@ -298,7 +298,7 @@ std::vector Manager::visitableLocations(const metkit::mars::MarsRequ std::set engines = Manager::engines(rq, all); - LOG_DEBUG_LIB(LibFdb5) << "Matching engines for request " << rq << " -> " << engines << std::endl; + LOG_DEBUG_LIB(LibFdb5) << "Matching engines for request " << rq << (all ? " ALL" : "") << " -> " << engines << std::endl; std::vector r; // union of all locations diff --git a/src/fdb5/database/MultiRetrieveVisitor.cc b/src/fdb5/database/MultiRetrieveVisitor.cc index cfed8c3a3..e1d1b8c70 100644 --- a/src/fdb5/database/MultiRetrieveVisitor.cc +++ b/src/fdb5/database/MultiRetrieveVisitor.cc @@ -11,18 +11,19 @@ #include "fdb5/database/MultiRetrieveVisitor.h" #include +#include +#include +#include -#include "eckit/config/Resource.h" +#include "eckit/log/Log.h" #include "fdb5/LibFdb5.h" +#include "fdb5/api/helpers/ListElement.h" #include "fdb5/database/DB.h" #include "fdb5/database/Key.h" -#include "fdb5/io/HandleGatherer.h" #include "fdb5/types/Type.h" #include "fdb5/types/TypesRegistry.h" - - namespace fdb5 { //---------------------------------------------------------------------------------------------------------------------- @@ -108,7 +109,7 @@ bool MultiRetrieveVisitor::selectDatum(const Key& key, const Key& full) { simplifiedKey.set(k->first, k->second); } - iterator_.emplace(ListElement({db_->key(), db_->indexKey(), simplifiedKey}, field.stableLocation(), field.timestamp())); + iterator_.emplace({db_->key(), db_->indexKey(), simplifiedKey, field.stableLocation(), field.timestamp()}); return true; } diff --git a/src/fdb5/remote/Handler.cc b/src/fdb5/remote/Handler.cc index 33fe984c6..3083bbcba 100644 --- a/src/fdb5/remote/Handler.cc +++ b/src/fdb5/remote/Handler.cc @@ -88,9 +88,12 @@ struct BaseHelper { }; struct ListHelper : public BaseHelper { - ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { - return fdb.list(request); - } + void extraDecode(eckit::Stream& s) { s >> level_; } + + ListIterator apiCall(FDB& fdb, const FDBToolRequest& request) const { return fdb.list(request, false, level_); } + +private: + int level_ {3}; }; struct InspectHelper : public BaseHelper { diff --git a/src/fdb5/tools/fdb-list.cc b/src/fdb5/tools/fdb-list.cc index 0c61129f0..85628912d 100644 --- a/src/fdb5/tools/fdb-list.cc +++ b/src/fdb5/tools/fdb-list.cc @@ -8,75 +8,75 @@ * does it submit to any jurisdiction. */ +#include +#include +#include #include +#include +#include +#include "eckit/exception/Exceptions.h" +#include "eckit/log/JSON.h" +#include "eckit/log/Log.h" #include "eckit/option/CmdArgs.h" -#include "eckit/config/Resource.h" #include "eckit/option/SimpleOption.h" -#include "eckit/option/VectorOption.h" -#include "eckit/option/CmdArgs.h" -#include "eckit/log/JSON.h" +#include "fdb5/LibFdb5.h" #include "metkit/hypercube/HyperCube.h" +#include "metkit/mars/MarsRequest.h" #include "fdb5/api/FDB.h" #include "fdb5/api/helpers/FDBToolRequest.h" -#include "fdb5/database/DB.h" -#include "fdb5/database/Index.h" +#include "fdb5/api/helpers/ListElement.h" +#include "fdb5/api/helpers/ListIterator.h" #include "fdb5/rules/Schema.h" #include "fdb5/tools/FDBVisitTool.h" using namespace eckit; using namespace eckit::option; -namespace fdb5 { -namespace tools { +namespace fdb5::tools { //---------------------------------------------------------------------------------------------------------------------- -class FDBList : public FDBVisitTool { - - public: // methods - - FDBList(int argc, char **argv) : - FDBVisitTool(argc, argv, "class,expver"), - location_(false), - timestamp_(false), - length_(false), - full_(false), - porcelain_(false), - json_(false) { - +class FDBList: public FDBVisitTool { +public: // methods + FDBList(int argc, char** argv): FDBVisitTool(argc, argv, "class,expver") { options_.push_back(new SimpleOption("location", "Also print the location of each field")); options_.push_back(new SimpleOption("timestamp", "Also print the timestamp when the field was indexed")); options_.push_back(new SimpleOption("length", "Also print the field size")); options_.push_back(new SimpleOption("full", "Include all entries (including masked duplicates)")); - options_.push_back(new SimpleOption("porcelain", "Streamlined and stable output for input into other tools")); + options_.push_back( + new SimpleOption("porcelain", + "Streamlined and stable output. Useful as input for other tools or scripts." + "Incompatible with options: location, timestamp, and length")); options_.push_back(new SimpleOption("json", "Output available fields in JSON form")); options_.push_back(new SimpleOption("compact", "Aggregate available fields in MARS requests")); + options_.push_back(new SimpleOption("depth", "Output entries up to 'depth' levels deep [1-3]")); } - private: // methods - - virtual void execute(const CmdArgs& args); - virtual void init(const CmdArgs &args); - - bool location_; - bool timestamp_; - bool length_; - bool full_; - bool porcelain_; - bool json_; - bool compact_; +private: // methods + void execute(const CmdArgs& args) override; + void init(const CmdArgs& args) override; + + bool location_ {false}; + bool timestamp_ {false}; + bool length_ {false}; + bool full_ {false}; + bool porcelain_ {false}; + bool json_ {false}; + bool compact_ {false}; + int depth_ {3}; }; +//---------------------------------------------------------------------------------------------------------------------- std::string keySignature(const fdb5::Key& key) { std::string signature; std::string separator; - for (auto k : key.keys()) { - signature += separator+k; - separator=":"; + for (auto&& k : key.keys()) { + signature += separator + k; + separator = ":"; } return signature; } @@ -86,31 +86,32 @@ void FDBList::init(const CmdArgs& args) { FDBVisitTool::init(args); - location_ = args.getBool("location", false); - timestamp_ = args.getBool("timestamp", false); - length_ = args.getBool("length", false); - full_ = args.getBool("full", false); - porcelain_ = args.getBool("porcelain", false); - json_ = args.getBool("json", false); - compact_ = args.getBool("compact", false); + location_ = args.getBool("location", location_); + timestamp_ = args.getBool("timestamp", timestamp_); + length_ = args.getBool("length", length_); + full_ = args.getBool("full", full_); + porcelain_ = args.getBool("porcelain", porcelain_); + json_ = args.getBool("json", json_); + compact_ = args.getBool("compact", compact_); + depth_ = args.getInt("depth", depth_); + + ASSERT(depth_ > 0 && depth_ < 4); if (json_) { + eckit::Log::debug() << "Setting porcelain=true" << '\n'; porcelain_ = true; - if (location_ || timestamp_ || length_) { - throw UserError("--json and --location/--timestamp/--length not compatible", Here()); - } + } + + if (porcelain_) { + if (location_) { throw UserError("--porcelain and --location are not compatible", Here()); } + if (timestamp_) { throw UserError("--porcelain and --timestamp are not compatible", Here()); } + if (length_) { throw UserError("--porcelain and --length are not compatible", Here()); } + if (compact_) { throw UserError("--porcelain and --compact are not compatible", Here()); } } if (compact_) { - if (location_) { - throw UserError("--compact and --location are not compatible", Here()); - } - if (full_) { - throw UserError("--compact and --full are not compatible", Here()); - } - if (porcelain_) { - throw UserError("--compact and --porcelain are not compatible", Here()); - } + if (location_) { throw UserError("--compact and --location are not compatible", Here()); } + if (full_) { throw UserError("--compact and --full are not compatible", Here()); } } /// @todo option ignore-errors @@ -122,7 +123,7 @@ void FDBList::execute(const CmdArgs& args) { std::unique_ptr json; if (json_) { - json.reset(new JSON(Log::info())); + json = std::make_unique(Log::info()); json->startList(); } @@ -135,14 +136,14 @@ void FDBList::execute(const CmdArgs& args) { } // If --full is supplied, then include all entries including duplicates. - auto listObject = fdb.list(request, !full_ && !compact_); + auto listObject = fdb.list(request, !full_ && !compact_, depth_); std::map>>> requests; ListElement elem; while (listObject.next(elem)) { if (compact_) { - std::vector keys = elem.key(); + const auto& keys = elem.keys(); ASSERT(keys.size() == 3); std::string treeAxes = keys[0]; @@ -165,19 +166,20 @@ void FDBList::execute(const CmdArgs& args) { it->second.emplace(signature, std::make_pair(keys[2].request(), std::unordered_set{keys[2]})); } } - } else { - if (json_) { - (*json) << elem; - } else { - if (porcelain_) { - elem.print(Log::info(), location_, false, false); - } else { - elem.print(Log::info(), location_, length_, timestamp_, ", "); - } - Log::info() << std::endl; - } + continue; } - } + + // JSON output + if (json) { + *json << elem; + continue; + } + + elem.print(Log::info(), location_, length_, timestamp_, ", "); + Log::info() << std::endl; + + } // while + if (compact_) { for (const auto& tree: requests) { for (const auto& leaf: tree.second) { @@ -200,20 +202,17 @@ void FDBList::execute(const CmdArgs& args) { } } // n.b. finding no data is not an error for fdb-list - } - if (json_) { - json->endList(); - } + } // requests + + if (json) { json->endList(); } } //---------------------------------------------------------------------------------------------------------------------- -} // namespace tools -} // namespace fdb5 +} // namespace fdb5::tools int main(int argc, char **argv) { fdb5::tools::FDBList app(argc, argv); return app.start(); } - diff --git a/src/fdb5/tools/fdb-patch.cc b/src/fdb5/tools/fdb-patch.cc index ad73af583..3a99320f3 100644 --- a/src/fdb5/tools/fdb-patch.cc +++ b/src/fdb5/tools/fdb-patch.cc @@ -144,7 +144,7 @@ void FDBPatch::execute(const CmdArgs& args) { // (n.b. listed key is broken down as-per the schema) Key key; - for (const Key& k : elem.keyParts_) { + for (const Key& k : elem.keys()) { for (const auto& kv : k) { key.set(kv.first, kv.second); } diff --git a/test.grib b/test.grib new file mode 100644 index 000000000..b1b371535 Binary files /dev/null and b/test.grib differ diff --git a/tests/fdb/api/ApiSpy.h b/tests/fdb/api/ApiSpy.h index c25efeebf..2c578157f 100644 --- a/tests/fdb/api/ApiSpy.h +++ b/tests/fdb/api/ApiSpy.h @@ -90,8 +90,9 @@ class ApiSpy : public fdb5::FDBBase { return fdb5::ListIterator(0); } - fdb5::ListIterator list(const fdb5::FDBToolRequest& request) override { + fdb5::ListIterator list(const fdb5::FDBToolRequest& /* request */, const int level) override { counts_.list += 1; + ASSERT(level == 3); return fdb5::ListIterator(0); } diff --git a/tests/fdb/api/test_fdb_c.cc b/tests/fdb/api/test_fdb_c.cc index 57c1f486c..7a9876570 100644 --- a/tests/fdb/api/test_fdb_c.cc +++ b/tests/fdb/api/test_fdb_c.cc @@ -8,9 +8,10 @@ * does it submit to any jurisdiction. */ -#include +#include +#include +#include -#include "eckit/config/Resource.h" #include "eckit/filesystem/PathName.h" #include "eckit/io/Buffer.h" #include "eckit/io/DataHandle.h" @@ -44,9 +45,10 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo EXPECT(err == FDB_SUCCESS); size_t level = 0; - for (auto key: keys) { - for (auto k1: key) { + for (const auto& key : keys) { + for (const auto& k1 : key) { int err = fdb_splitkey_next_metadata(sk, &k, &v, checkLevel ? &l : nullptr); + std::cerr << "k=" << k << " v=" << v << " l=" << l << std::endl; EXPECT(err == FDB_SUCCESS); EXPECT(k1.first == k); EXPECT(k1.second == v); @@ -58,11 +60,12 @@ void key_compare(const std::vector& keys, fdb_listiterator_t *it, boo } err = fdb_splitkey_next_metadata(sk, &k, &v, &l); EXPECT(err == FDB_ITERATION_COMPLETE); - + err = fdb_delete_splitkey(sk); } CASE( "fdb_c - archive & list" ) { + const int depth = 3; size_t length; DataHandle *dh; @@ -110,10 +113,10 @@ CASE( "fdb_c - archive & list" ) { const char **item= new const char*; fdb_listiterator_t* it; - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); int err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + const char *uri; size_t off, attr_len; @@ -135,7 +138,7 @@ CASE( "fdb_c - archive & list" ) { fdb_request_add1(request, "param", "139"); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); @@ -154,17 +157,17 @@ CASE( "fdb_c - archive & list" ) { EXPECT(FDB_SUCCESS == fdb_flush(fdb)); fdb_request_add1(request, "levelist", "400"); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); fdb_request_add1(request, "param", "138"); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + fdb_listiterator_attrs(it, &uri, &off, &attr_len); EXPECT(attr_len == 3280398); @@ -184,7 +187,7 @@ CASE( "fdb_c - archive & list" ) { const char* values[] = {"400", "300"}; fdb_request_add(request, "levelist", values, 2); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); @@ -220,6 +223,7 @@ CASE( "fdb_c - archive & list" ) { #if fdb5_HAVE_GRIB CASE( "fdb_c - multiple archive & list" ) { + const int depth = 3; size_t length1, length2, length3; DataHandle *dh; @@ -320,7 +324,7 @@ CASE( "fdb_c - multiple archive & list" ) { const char **item= new const char*; bool exist; fdb_listiterator_t* it; - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); int err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); @@ -331,7 +335,7 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_delete_listiterator(it); fdb_request_add1(request, "step", "1"); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); @@ -339,21 +343,94 @@ CASE( "fdb_c - multiple archive & list" ) { fdb_request_add1(request, "step", "0"); const char* values[] = {"400", "300"}; fdb_request_add(request, "levelist", values, 2); - fdb_list(fdb, request, &it, true); + fdb_list(fdb, request, &it, true, depth); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k1, it); err = fdb_listiterator_next(it); ASSERT(err == FDB_SUCCESS); - + key_compare(k2, it); err = fdb_listiterator_next(it); ASSERT(err == FDB_ITERATION_COMPLETE); fdb_delete_listiterator(it); } + +CASE("fdb_c - list depth=1,2,3") { + fdb_handle_t* fdb = nullptr; + fdb_new_handle(&fdb); + + std::vector key300d1 { + {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + }; + + std::vector key300d2 { + {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + {{"type", "an"}, {"levtype", "pl"}}, + }; + + std::vector key300d3 { + {{"class", "rd"}, {"expver", "xxxx"}, {"stream", "oper"}, {"date", "20191110"}, {"time", "0000"}, {"domain", "g"}}, + {{"type", "an"}, {"levtype", "pl"}}, + {{"step", "0"}, {"levelist", "300"}, {"param", "138"}}, + }; + + fdb_request_t* request = nullptr; + fdb_new_request(&request); + fdb_request_add1(request, "domain", "g"); + fdb_request_add1(request, "stream", "oper"); + fdb_request_add1(request, "levtype", "pl"); + fdb_request_add1(request, "levelist", "300"); + fdb_request_add1(request, "date", "20191110"); + fdb_request_add1(request, "time", "0000"); + fdb_request_add1(request, "step", "0"); + fdb_request_add1(request, "param", "138"); + fdb_request_add1(request, "class", "rd"); + fdb_request_add1(request, "type", "an"); + fdb_request_add1(request, "expver", "xxxx"); + + { // depth=1 + fdb_listiterator_t* iter = nullptr; + fdb_list(fdb, request, &iter, true, 1); + int err = fdb_listiterator_next(iter); + ASSERT(err == FDB_SUCCESS); + + key_compare(key300d1, iter); + + err = fdb_listiterator_next(iter); + ASSERT(err == FDB_ITERATION_COMPLETE); + fdb_delete_listiterator(iter); + } + + { // depth=2 + fdb_listiterator_t* iter = nullptr; + fdb_list(fdb, request, &iter, true, 2); + int err = fdb_listiterator_next(iter); + ASSERT(err == FDB_SUCCESS); + + key_compare(key300d2, iter); + + err = fdb_listiterator_next(iter); + ASSERT(err == FDB_ITERATION_COMPLETE); + fdb_delete_listiterator(iter); + } + + { // depth=3 + fdb_listiterator_t* iter = nullptr; + fdb_list(fdb, request, &iter, true, 3); + int err = fdb_listiterator_next(iter); + ASSERT(err == FDB_SUCCESS); + + key_compare(key300d3, iter); + + err = fdb_listiterator_next(iter); + ASSERT(err == FDB_ITERATION_COMPLETE); + fdb_delete_listiterator(iter); + } +} #endif CASE( "fdb_c - retrieve bad request" ) { @@ -480,7 +557,7 @@ CASE( "fdb_c - expand" ) { size_t numValues; char** values; - + fdb_request_get(request, "date", &values, &numValues); EXPECT_EQUAL(numValues, 2); EXPECT_EQUAL(0, strncmp(values[0], "20191110", 8)); diff --git a/tests/fdb/test_fdb5_service.cc b/tests/fdb/test_fdb5_service.cc index 7c23287a4..a7c13a08c 100644 --- a/tests/fdb/test_fdb5_service.cc +++ b/tests/fdb/test_fdb5_service.cc @@ -18,15 +18,12 @@ #include #include "eckit/io/DataHandle.h" -#include "eckit/io/FileHandle.h" #include "eckit/io/MemoryHandle.h" -#include "eckit/io/MultiHandle.h" #include "eckit/runtime/Main.h" #include "eckit/types/Types.h" #include "eckit/utils/Translator.h" #include "metkit/mars/MarsRequest.h" -#include "metkit/mars/MarsExpension.h" #include "metkit/mars/TypeAny.h" #include "fdb5/database/Key.h" @@ -134,9 +131,9 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "60"); - EXPECT(!iter.next(el)); - } + EXPECT(el.combinedKey().get("step") == "60"); + EXPECT(!iter.next(el)); + } key.set("step","2"); key.set("stepunits","h"); @@ -149,8 +146,8 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "2"); - EXPECT(!iter.next(el)); + EXPECT(el.combinedKey().get("step") == "2"); + EXPECT(!iter.next(el)); } req.setValuesTyped(new metkit::mars::TypeAny("step"), std::vector{"2","60"}); @@ -159,11 +156,11 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "2"); - EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "60"); - EXPECT(!iter.next(el)); - } + EXPECT(el.combinedKey().get("step") == "2"); + EXPECT(iter.next(el)); + EXPECT(el.combinedKey().get("step") == "60"); + EXPECT(!iter.next(el)); + } // sub-hourly data are not yet supported in metkit /* key.set("step","30"); @@ -177,7 +174,7 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "30m"); + EXPECT(el.key().get("step") == "30m"); EXPECT(!iter.next(el)); } @@ -189,9 +186,9 @@ CASE ( "test_fdb_stepunit_archive" ) { fdb5::ListIterator iter = fdb.list(r, true); fdb5::ListElement el; EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "30m"); + EXPECT(el.key().get("step") == "30m"); EXPECT(iter.next(el)); - EXPECT(el.combinedKey().get("step") == "2"); + EXPECT(el.key().get("step") == "2"); EXPECT(!iter.next(el)); }*/ } @@ -306,7 +303,7 @@ CASE ( "test_fdb_service" ) { fdb5::ListElement el; EXPECT(iter.next(el)); - eckit::PathName path = el.location().uri().path().dirName(); + eckit::PathName path = el.uri().path().dirName(); DIR* dirp = ::opendir(path.asString().c_str()); struct dirent* dp; @@ -377,8 +374,8 @@ CASE ( "test_fdb_service_subtoc" ) { eckit::LocalConfiguration userConf; userConf.set("useSubToc", true); - - fdb5::Config expanded = fdb5::Config().expandConfig(); + + fdb5::Config expanded = fdb5::Config().expandConfig(); fdb5::Config config(expanded, userConf); @@ -489,7 +486,7 @@ CASE ( "test_fdb_service_subtoc" ) { fdb5::ListElement el; EXPECT(iter.next(el)); - eckit::PathName path = el.location().uri().path().dirName(); + eckit::PathName path = el.uri().path().dirName(); DIR* dirp = ::opendir(path.asString().c_str()); struct dirent* dp; diff --git a/tests/fdb/tools/CMakeLists.txt b/tests/fdb/tools/CMakeLists.txt index 6ff2df23a..308c35b66 100644 --- a/tests/fdb/tools/CMakeLists.txt +++ b/tests/fdb/tools/CMakeLists.txt @@ -3,7 +3,7 @@ list( APPEND fdb_tools_tests fdb_axes ) foreach( _t ${fdb_tools_tests} ) - + configure_file( ${_t}.sh.in ${_t}.sh @ONLY ) ecbuild_add_test( @@ -14,3 +14,5 @@ foreach( _t ${fdb_tools_tests} ) endforeach() add_subdirectory( auxiliary ) + +add_subdirectory( list ) diff --git a/tests/fdb/tools/list/CMakeLists.txt b/tests/fdb/tools/list/CMakeLists.txt new file mode 100644 index 000000000..37ea2b3a6 --- /dev/null +++ b/tests/fdb/tools/list/CMakeLists.txt @@ -0,0 +1,29 @@ +# get test data for the list tests +ecbuild_get_test_multidata( TARGET get_fdb_test_data_list LABELS list NAMES "od.oper.grib" ) + +# list of tests +list( APPEND fdb_tools_list_tests + all + all_full + location + masking + minimum_keys + porcelain + ranges + depth +) + +# add the tests +foreach( _test ${fdb_tools_list_tests} ) + + configure_file( list_${_test}.sh.in list_${_test}.sh @ONLY ) + + ecbuild_add_test( + TARGET test_fdb_tools_list_${_test} + TYPE SCRIPT + LABELS list + CONDITION HAVE_FDB_BUILD_TOOLS + TEST_DEPENDS get_fdb_test_data_list + COMMAND list_${_test}.sh ) + +endforeach( ) diff --git a/tests/fdb/tools/list/list_all.sh.in b/tests/fdb/tools/list/list_all.sh.in new file mode 100755 index 000000000..f54007e15 --- /dev/null +++ b/tests/fdb/tools/list/list_all.sh.in @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -cE "$1" out) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=all +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +try rm -rf $test_name +try mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" + +######################################################################################################################## + +# Ensure that listing finds the correct data, but excludes duplicates. + +# Set up the data for the test + +try grib_set -s class=rd,expver=xxxx,type=fc,step=0 "$src_data" data.xxxx.0.grib +try grib_set -s class=rd,expver=xxxx,type=fc,step=1 "$src_data" data.xxxx.1.grib +try grib_set -s class=rd,expver=xxxx,type=fc,step=2 "$src_data" data.xxxx.2.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=0 "$src_data" data.xxxy.0.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=1 "$src_data" data.xxxy.1.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=2 "$src_data" data.xxxy.2.grib + +# Set up some regexes for later testing + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_x2="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}" +regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_y2="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=2,levelist=[0-9]+,param=[0-9]+}" + +# We loop over all these tests twice. We should get the same listing results both times +# as the default behaviour is to only show the data that would be retrieved (i.e. that matches a MARS request.) + +sum=0 +for i in 1 2; do + + echo "============ Loop $i ============" + + try fdb-write data.xxxx.0.grib + try fdb-list --all --minimum-keys="" --porcelain | tee out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + line_count $sum + grep_count "$regex_x1" 0 + grep_count "$regex_x2" 0 + grep_count "$regex_y0" 0 + grep_count "$regex_y1" 0 + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + + try fdb-write data.xxxy.0.grib + try fdb-list --all --minimum-keys="" --porcelain | tee out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + line_count $sum + grep_count "$regex_x1" 0 + grep_count "$regex_x2" 0 + grep_count "$regex_y1" 0 + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + + + try fdb-write data.xxxx.1.grib + try fdb-list --all --minimum-keys="" --porcelain | tee out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + line_count $sum + grep_count "$regex_x2" 0 + grep_count "$regex_y1" 0 + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + grep_count "$regex_x1" 24 + + try fdb-write data.xxxy.1.grib + try fdb-list --all --minimum-keys="" --porcelain | tee out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + line_count $sum + grep_count "$regex_x2" 0 + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + grep_count "$regex_x1" 24 + grep_count "$regex_y1" 24 + + + try fdb-write data.xxxx.2.grib + try fdb-list --all --minimum-keys="" --porcelain | tee out + if [[ $i = 1 ]]; then + sum=$((sum + 24)) + line_count $sum + grep_count "$regex_y2" 0 + fi + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + grep_count "$regex_x1" 24 + grep_count "$regex_y1" 24 + grep_count "$regex_x2" 24 + + try fdb-write data.xxxy.2.grib + try fdb-list --all --minimum-keys="" --porcelain | tee out + [[ $i = 1 ]] && sum=$((sum + 24)) + line_count $sum + grep_count "$regex_x0" 24 + grep_count "$regex_y0" 24 + grep_count "$regex_x1" 24 + grep_count "$regex_y1" 24 + grep_count "$regex_x2" 24 + grep_count "$regex_y2" 24 + +done + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_all_full.sh.in b/tests/fdb/tools/list/list_all_full.sh.in new file mode 100755 index 000000000..b20ed0f0e --- /dev/null +++ b/tests/fdb/tools/list/list_all_full.sh.in @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -cEE "$1" out) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=all_full +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +try rm -rf $test_name +try mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" + +######################################################################################################################## + +# Ensure that listing finds the correct data, but excludes duplicates. +# This is the same data as in the all test, but with a full enumeration + +# Set up the data for the test + +try grib_set -s class=rd,expver=xxxx,type=fc,step=0 "$src_data" data.xxxx.0.grib +try grib_set -s class=rd,expver=xxxx,type=fc,step=1 "$src_data" data.xxxx.1.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=0 "$src_data" data.xxxy.0.grib +try grib_set -s class=rd,expver=xxxy,type=fc,step=1 "$src_data" data.xxxy.1.grib + +# Set up some regexes for later testing + +regex_x0="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_x1="{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" +regex_y0="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" +regex_y1="{class=rd,expver=xxxy,stream=oper,date=[0-9]+,time=(12|00)00,domain=g}{type=fc,levtype=pl}{step=1,levelist=[0-9]+,param=[0-9]+}" + +# We loop over all these tests twice. We should get the same listing results both times +# as the default behaviour is to only show the data that would be retrieved (i.e. that matches a MARS request.) + +for i in 0 1; do + + echo "============ Loop $i ============" + + try fdb-write data.xxxx.0.grib + try fdb-list --all --minimum-keys="" --porcelain --full | tee out + line_count $((24+(i*96))) + grep_count "$regex_x0" $((24+(24*i))) + grep_count "$regex_x1" $((0+(24*i))) + grep_count "$regex_y0" $((0+(24*i))) + grep_count "$regex_y1" $((0+(24*i))) + + try fdb-write data.xxxy.0.grib + try fdb-list --all --minimum-keys="" --porcelain --full | tee out + line_count $((48+(i*96))) + grep_count "$regex_x0" $((24+(24*i))) + grep_count "$regex_x1" $((0+(24*i))) + grep_count "$regex_y0" $((24+(24*i))) + grep_count "$regex_y1" $((0+(24*i))) + + try fdb-write data.xxxx.1.grib + try fdb-list --all --minimum-keys="" --porcelain --full | tee out + line_count $((72+(i*96))) + grep_count "$regex_x0" $((24+(24*i))) + grep_count "$regex_x1" $((24+(24*i))) + grep_count "$regex_y0" $((24+(24*i))) + grep_count "$regex_y1" $((0+(24*i))) + + try fdb-write data.xxxy.1.grib + try fdb-list --all --minimum-keys="" --porcelain --full | tee out + line_count $((96+(i*96))) + grep_count "$regex_x0" $((24+(24*i))) + grep_count "$regex_x1" $((24+(24*i))) + grep_count "$regex_y0" $((24+(24*i))) + grep_count "$regex_y1" $((24+(24*i))) + +done + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_depth.sh.in b/tests/fdb/tools/list/list_depth.sh.in new file mode 100755 index 000000000..70ba3a833 --- /dev/null +++ b/tests/fdb/tools/list/list_depth.sh.in @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +######################################################################################################################## + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=depth +src_data=x.grib +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +try rm -rf $test_name +try mkdir -p $test_name/localroot + +try cd $test_name + +for f in local.yaml $src_data; do + try cp "$src_dir/$f" ./ +done + +######################################################################################################################## +# Create a grib file with 4 steps and populate FDB + +try grib_set -s step=0 $src_data data.0.grib +try grib_set -s step=6 $src_data data.6.grib +try grib_set -s step=9 $src_data data.9.grib + +for f in *.grib; do + try grib_set -s type=an "$f" "data.an.${f#*.}" +done + +for f in *.grib; do + try cat "$f" >> "data.$test_name.grib" +done + +try fdb-write "data.$test_name.grib" + +work_dir=$(pwd) +echo "Working directory: $work_dir" + +######################################################################################################################## + +echo "Test: all" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc}{step=9,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" +out=$(try fdb-list --all --minimum-keys= --porcelain) +try test "$exp" = "$out" + +echo "Test: date depth=1" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" +out=$(try fdb-list date=20201102 --minimum-keys="" --porcelain --depth=1) +try test "$exp" = "$out" + +echo "Test: date depth=2" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=fc,levtype=sfc} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" +out=$(try fdb-list date=20201102 --minimum-keys="" --porcelain --depth=2) +try test "$exp" = "$out" + +echo "Test: cf depth=1" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}" +out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=1) +try test "$exp" = "$out" + +echo "Test: cf depth=2" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}" +out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=2) +try test "$exp" = "$out" + +echo "Test: cf depth=3" +exp="{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=0,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=12,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=6,param=166} +{class=rd,expver=xxxx,stream=oper,date=20201102,time=0000,domain=g}{type=an,levtype=sfc}{step=9,param=166}" +out=$(try fdb-list type=an --minimum-keys="" --porcelain --depth=3) +try test "$exp" = "$out" + + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/*data.*.grib diff --git a/tests/fdb/tools/list/list_location.sh.in b/tests/fdb/tools/list/list_location.sh.in new file mode 100755 index 000000000..fe3e058bf --- /dev/null +++ b/tests/fdb/tools/list/list_location.sh.in @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -cE "$1" out) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=location +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +try rm -rf $test_name +try mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" + +######################################################################################################################## + +# Check that we can obtain the location of the data + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib +try grib_ls -m data.xxxx.grib +try fdb-write data.xxxx.grib + +echo "Test: --all --minimum-keys= --porcelain --full" +try fdb-list --all --minimum-keys= --porcelain --full | tee out +line_count 24 + + +echo "Test: Specify the location flag" +try fdb-list class=rd,expver=xxxx,time=0000 --location | tee out +grep_count "localroot/rd:xxxx" 12 + + +echo "Test: without the location flag, none of the location info is included" +try fdb-list class=rd,expver=xxxx,time=0000 --porcelain | tee out +line_count 12 +grep_count "{class=rd,expver=xxxx,stream=oper,date=[0-9]+,time=0000,domain=g}{type=an,levtype=pl}{step=0,levelist=[0-9]+,param=[0-9]+}" 12 + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_masking.sh.in b/tests/fdb/tools/list/list_masking.sh.in new file mode 100755 index 000000000..5c5e8ec8e --- /dev/null +++ b/tests/fdb/tools/list/list_masking.sh.in @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +######################################################################################################################## + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=masking +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +try rm -rf $test_name +try mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" + +######################################################################################################################## + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.d1.grib +try grib_set -s class=rd,expver=xxxx,date=20240912 "$src_data" data.xxxx.d2.grib +try grib_ls -m data.xxxx.d1.grib + +try fdb-write data.xxxx.d1.grib +try fdb-write data.xxxx.d2.grib +# write the data twice +try fdb-write data.xxxx.d1.grib +try fdb-write data.xxxx.d2.grib + +echo "Test: Incorrect number of entries written" +try fdb-list --all --minimum-keys="" --porcelain | tee out +line_count 48 + +echo "Test: Incorrect number of entries written" +try fdb-list --all --minimum-keys="" --porcelain --full | tee out +line_count 96 + +echo "Test: Entries should be masked without --full" +try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out +line_count 48 + +echo "Test: All entries should be visible with --full" +try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=155/138,levelist=300/400/500/700/850/1000 --porcelain --full | tee out +line_count 96 + +echo "Test: Entries should be masked without full" +try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain | tee out +line_count 8 + +echo "Test: All entries should be visible with --full" +try fdb-list class=rd,expver=xxxx,date=20240911/20240912,stream=oper,type=an,levtype=pl,param=130/138,levelist=300/123/1000 --porcelain --full | tee out +line_count 16 + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_minimum_keys.sh.in b/tests/fdb/tools/list/list_minimum_keys.sh.in new file mode 100755 index 000000000..e8aad4f0a --- /dev/null +++ b/tests/fdb/tools/list/list_minimum_keys.sh.in @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +######################################################################################################################## + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=minimum_keys +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +try rm -rf $test_name +try mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" + +######################################################################################################################## + +# Ensure that we can wipe specified databases, but only when fully specified + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib +try grib_ls -m data.xxxx.grib +try fdb-write data.xxxx.grib + + +echo "Test: check all output" +try fdb-list --all --full --minimum-keys= --porcelain | tee out +line_count 24 + + +echo "Test: nothing happens if any of the keys are missing" +for invalid_key in class=rd expver=xxxx; do + fdb-list $invalid_key + rc=$? + [[ $rc = 0 ]] && die "Should not have succeeded: $invalid_key" +done + + +echo "Test: supplying the key" +try fdb-list class=rd,expver=xxxx --porcelain | tee out +line_count 24 + + +echo "Test: changing the minimum keys" +for invalid_key in class=rd,expver=xxxx class=rd,time=1200 expver=xxxx,time=1200; do + fdb-list --minimum-keys="class,expver,time" $invalid_key + rc=$? + [[ $rc = 0 ]] && die "Should not have succeeded: $invalid_key" +done + + +echo "Test: the purge works now this is supplied" +try fdb-list --minimum-keys="class,expver,time" class=rd,expver=xxxx,time=1200 --porcelain | tee out +line_count 12 + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_porcelain.sh.in b/tests/fdb/tools/list/list_porcelain.sh.in new file mode 100755 index 000000000..75a6d9440 --- /dev/null +++ b/tests/fdb/tools/list/list_porcelain.sh.in @@ -0,0 +1,121 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +grep_count() { + [[ $# -eq 2 ]] || die "grep_count requires 2; regex and expected count" + val=$(grep -cE "$1" out) + [[ $val -eq $2 ]] || die "Incorrect count [$val != $2] for regex [$1]" +} + +######################################################################################################################## + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=porcelain +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +try rm -rf $test_name +try mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" + +######################################################################################################################## + +# If we don't specify --porcelain, then the expanded request is prepended to the output request. +# Measure the difference! + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.grib +try grib_ls -m data.xxxx.grib +try fdb-write data.xxxx.grib + +tab=$'\t' # macos doesn't support '\t' in grep + +echo "Test all output" + +try fdb-list --all --minimum-keys="" --porcelain --full | tee out +line_count 24 + + +echo "Test normal output" + +try fdb-list class=rd,expver=xxxx | tee out +grep_count "{class=rd,expver=xxxx" 24 +grep_count "^retrieve,$" 1 +grep_count "^${tab}.*=.*" 2 +grep_count "^${tab}class=rd" 1 +grep_count "^${tab}expver=xxxx" 1 + + +echo "Test porcelain output" + +try fdb-list class=rd,expver=xxxx --porcelain | tee out +line_count 24 +grep_count "{class=rd,expver=xxxx" 24 + + +echo "Test expansion of date and param without porcelain" + +try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=t | tee out +grep_count "{class=rd,expver=xxxx" 0 +grep_count "^retrieve,$" 1 +grep_count "^${tab}.*=.*" 7 +grep_count "^${tab}class=rd" 1 +grep_count "^${tab}expver=xxxx" 1 +grep_count "^${tab}param=130" 1 +grep_count "^${tab}date=20240911" 1 + + +echo "Test expansion of date and param with porcelain" + +try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=t --porcelain | tee out +line_count 0 +[[ "$(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | wc -l)" -eq 0 ]] || die "Incorrect number of entries reported" + + +echo "Test expansion of date and param without porcelain" + +try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=138 | tee out +grep_count "^retrieve,$" 1 +grep_count "^${tab}.*=.*" 7 +grep_count "^${tab}class=rd" 1 +grep_count "^${tab}expver=xxxx" 1 +grep_count "^${tab}param=138" 1 +grep_count "^${tab}date=20240911" 1 +[[ $(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | grep param=138 | wc -l) -eq 12 ]] || die "Incorrect number of entries reported" + + +echo "Test expansion of date and param with porcelain" + +try fdb-list class=rd,expver=xxxx,date=20240911,stream=oper,type=an,levtype=pl,param=138 --porcelain | tee out +line_count 12 +[[ $(grep '{class=rd,expver=xxxx' out | grep 'date=20240911' | grep param=138 | wc -l) -eq 12 ]] || die "Incorrect number of entries reported" + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/list_ranges.sh.in b/tests/fdb/tools/list/list_ranges.sh.in new file mode 100755 index 000000000..0cd71222f --- /dev/null +++ b/tests/fdb/tools/list/list_ranges.sh.in @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +set -ux + +yell() { printf "$(basename "$0"): \033[0;31m!!! %s !!!\033[0m\\n" "$*" >&2; } +die() { yell "$*"; exit 1; } +try() { "$@" || die "Errored HERE => '$*'"; } + +line_count() { + [[ $# -eq 1 ]] || die "line_count requires 1 argument; expected line count" + val=$(wc -l < out) && val=$((val + 0)) + [[ $val -eq $1 ]] || die "Incorrect count => [$val != $1]" +} + +######################################################################################################################## + +export PATH=@CMAKE_BINARY_DIR@/bin:$PATH +export FDB5_CONFIG_FILE="local.yaml" +export FDB_HOME=@PROJECT_BINARY_DIR@ + +test_name=ranges +src_data="od.oper.grib" +src_dir=@CMAKE_CURRENT_SOURCE_DIR@ +bin_dir=@CMAKE_CURRENT_BINARY_DIR@ + +######################################################################################################################## + +echo "running test '$test_name' on $(hostname)" + +try cd $bin_dir + +try rm -rf $test_name +try mkdir -p $test_name/localroot + +try cd $test_name + +try cp "$src_dir/local.yaml" ./ +try ln -s "$bin_dir/$src_data" ./ + +work_dir=$(pwd) +echo "Working directory: $work_dir" + +######################################################################################################################## + +# Ensure that we can wipe specified databases with ranges in the requests +# --> Ensures the MARS requests are working correctly + +try grib_set -s class=rd,expver=xxxx "$src_data" data.xxxx.d1.grib +try grib_set -s class=rd,expver=xxxx,date=20240912 "$src_data" data.xxxx.d2.grib +try grib_set -s class=rd,expver=xxxx,date=20241001 "$src_data" data.xxxx.d3.grib +try grib_ls -m data.xxxx.d1.grib + +try fdb-write data.xxxx.d1.grib +try fdb-write data.xxxx.d2.grib +try fdb-write data.xxxx.d3.grib + +try fdb-list --all --minimum-keys="" --porcelain --full | tee out +line_count 72 + +try fdb-list class=rd,expver=xxxx --porcelain | tee out +line_count 72 + +try fdb-list class=rd,expver=xxxx,date=20240911 --porcelain | tee out +line_count 24 + +try fdb-list class=rd,expver=xxxx,date=20240911/20240912 --porcelain | tee out +line_count 48 + +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001 --porcelain | tee out +line_count 72 + +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001/20060101,stream=oper,type=an,levtype=pl,param=60 --porcelain | tee out +line_count 0 + +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=155 --porcelain | tee out +line_count 36 + +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155 --porcelain | tee out +line_count 36 + +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138 --porcelain | tee out +line_count 72 + +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/400/500/700/850/1000 --porcelain | tee out +line_count 72 + +try fdb-list class=rd,expver=xxxx,date=20240911/20240912/20241001,stream=oper,type=an,levtype=pl,param=60/155/138,levelist=300/123/1000 --porcelain | tee out +line_count 24 + +echo "cleanup" +try rm -rf "$work_dir/localroot" "$work_dir"/data.*.grib diff --git a/tests/fdb/tools/list/local.yaml b/tests/fdb/tools/list/local.yaml new file mode 100644 index 000000000..8e43d2e8f --- /dev/null +++ b/tests/fdb/tools/list/local.yaml @@ -0,0 +1,6 @@ +--- +type: local +engine: toc +spaces: + - roots: + - path: ./localroot diff --git a/tests/fdb/tools/list/x.grib b/tests/fdb/tools/list/x.grib new file mode 120000 index 000000000..635e33b3e --- /dev/null +++ b/tests/fdb/tools/list/x.grib @@ -0,0 +1 @@ +../../../regressions/FDB-307/x.grib \ No newline at end of file diff --git a/tests/fdb/type/test_toKey.cc b/tests/fdb/type/test_toKey.cc index 5819a0416..ee46109d3 100644 --- a/tests/fdb/type/test_toKey.cc +++ b/tests/fdb/type/test_toKey.cc @@ -124,6 +124,8 @@ CASE( "Step & ClimateDaily - expansion" ) { key.set("levelist", "50"); key.set("param", "129.128"); + EXPECT(key.valuesToString() == "20210427:dacl:0000:ei:7799:g:pb:pl:2-12:99:100:50:129.128"); + fdb5::Archiver archiver; fdb5::ArchiveVisitor visitor(archiver, key, data, 4); config.schema().expand(key, visitor); @@ -131,8 +133,7 @@ CASE( "Step & ClimateDaily - expansion" ) { EXPECT(key.canonicalValue("date") == "0427"); EXPECT(key.canonicalValue("time") == "0000"); - - std::cout << key.valuesToString() << std::endl; + EXPECT(key.canonicalValue("step") == "2-12"); EXPECT(key.valuesToString() == "0427:dacl:0000:ei:7799:g:pb:pl:2-12:99:100:50:129.128");