From 1d72cb4305760098ca3680ffc599d5e6e574adc2 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 25 Dec 2024 16:23:39 -0800 Subject: [PATCH 01/19] Add unit test for HDFIO::getH5ObjectType --- src/io/hdf5/HDF5IO.hpp | 2 +- tests/testHDF5IO.cpp | 32 +++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/io/hdf5/HDF5IO.hpp b/src/io/hdf5/HDF5IO.hpp index 575dbcc..a2f862d 100644 --- a/src/io/hdf5/HDF5IO.hpp +++ b/src/io/hdf5/HDF5IO.hpp @@ -311,7 +311,7 @@ class HDF5IO : public BaseIO /** * @brief Returns the HDF5 type of object at a given path. * @param path The location in the file of the object. - * @return The type of object at the given path. + * @return The type of object at the given path. A negative value indicates that an error occurred. */ H5O_type_t getH5ObjectType(const std::string& path) const; diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 6d02537..1ba3046 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -508,4 +508,34 @@ TEST_CASE("readDataset", "[hdf5io]") hdf5io->close(); } -} \ No newline at end of file +} + +TEST_CASE("getH5ObjectType", "[hdf5io]") +{ + // create and open file + std::string filename = getTestFilePath("test_getH5ObjectType.h5"); + IO::HDF5::HDF5IO hdf5io(filename); + hdf5io.open(); + + SECTION("group") + { + hdf5io.createGroup("/group"); + REQUIRE(hdf5io.getH5ObjectType("/group") == H5O_TYPE_GROUP); + } + + SECTION("dataset") + { + std::vector testData = {1, 2, 3, 4, 5}; + std::string dataPath = "/dataset"; + hdf5io.createArrayDataSet(BaseDataType::I32, SizeArray {0}, SizeArray {1}, dataPath); + REQUIRE(hdf5io.getH5ObjectType(dataPath) == H5O_TYPE_DATASET); + } + + SECTION("non_existent_object") + { + REQUIRE(hdf5io.getH5ObjectType("/non_existent") < 0); + } + + // close file + hdf5io.close(); +} From 2f048e3aa6500b0173ed9cfccebc58ff5f156737 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 25 Dec 2024 17:04:44 -0800 Subject: [PATCH 02/19] Fix behavior of HDF5IO::getH5ObjectType when object does not exist --- src/io/hdf5/HDF5IO.cpp | 20 +++++++--- src/io/hdf5/HDF5IO.hpp | 3 +- tests/testHDF5IO.cpp | 89 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 103 insertions(+), 9 deletions(-) diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index bfc3eef..4c638ca 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -876,16 +876,24 @@ std::unique_ptr HDF5IO::createArrayDataSet( H5O_type_t HDF5IO::getH5ObjectType(const std::string& path) const { -#if H5_VERSION_GE(1, 12, 0) - // get whether path is a dataset or group H5O_info_t objInfo; // Structure to hold information about the object - H5Oget_info_by_name( + herr_t status; + +#if H5_VERSION_GE(1, 12, 0) + status = H5Oget_info_by_name( m_file->getId(), path.c_str(), &objInfo, H5O_INFO_BASIC, H5P_DEFAULT); #else - // get whether path is a dataset or group - H5O_info_t objInfo; // Structure to hold information about the object - H5Oget_info_by_name(m_file->getId(), path.c_str(), &objInfo, H5P_DEFAULT); + status = + H5Oget_info_by_name(m_file->getId(), path.c_str(), &objInfo, H5P_DEFAULT); #endif + + // Check if the object exists + if (status < 0) { + // Return a special value indicating the object does not exist + return H5O_TYPE_UNKNOWN; + } + + // Get the object type H5O_type_t objectType = objInfo.type; return objectType; diff --git a/src/io/hdf5/HDF5IO.hpp b/src/io/hdf5/HDF5IO.hpp index a2f862d..ad026ce 100644 --- a/src/io/hdf5/HDF5IO.hpp +++ b/src/io/hdf5/HDF5IO.hpp @@ -311,7 +311,8 @@ class HDF5IO : public BaseIO /** * @brief Returns the HDF5 type of object at a given path. * @param path The location in the file of the object. - * @return The type of object at the given path. A negative value indicates that an error occurred. + * @return The type of object at the given path. H5O_TYPE_UNKNOWN indicates + * that the object does not exist (or is of an unknown type). */ H5O_type_t getH5ObjectType(const std::string& path) const; diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 1ba3046..2881886 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -527,15 +527,100 @@ TEST_CASE("getH5ObjectType", "[hdf5io]") { std::vector testData = {1, 2, 3, 4, 5}; std::string dataPath = "/dataset"; - hdf5io.createArrayDataSet(BaseDataType::I32, SizeArray {0}, SizeArray {1}, dataPath); + hdf5io.createArrayDataSet( + BaseDataType::I32, SizeArray {0}, SizeArray {1}, dataPath); REQUIRE(hdf5io.getH5ObjectType(dataPath) == H5O_TYPE_DATASET); } SECTION("non_existent_object") { - REQUIRE(hdf5io.getH5ObjectType("/non_existent") < 0); + REQUIRE(hdf5io.getH5ObjectType("/non_existent") == H5O_TYPE_UNKNOWN); } // close file hdf5io.close(); } + +/* TEST_CASE("HDF5IO::getNativeType", "[hdf5io]") +{ + SECTION("Integer Types") + { + // Test for T_I8 + BaseDataType typeI8(BaseDataType::T_I8, 1); + H5::DataType nativeTypeI8 = IO::HDF5::HDF5IO::getNativeType(typeI8); + REQUIRE(nativeTypeI8.getId() == H5::PredType::NATIVE_INT8.getId()); + + // Test for T_I16 + BaseDataType typeI16(BaseDataType::T_I16, 1); + H5::DataType nativeTypeI16 = IO::HDF5::HDF5IO::getNativeType(typeI16); + REQUIRE(nativeTypeI16.getId() == H5::PredType::NATIVE_INT16.getId()); + + // Test for T_I32 + BaseDataType typeI32(BaseDataType::T_I32, 1); + H5::DataType nativeTypeI32 = IO::HDF5::HDF5IO::getNativeType(typeI32); + REQUIRE(nativeTypeI32.getId() == H5::PredType::NATIVE_INT32.getId()); + + // Test for T_I64 + BaseDataType typeI64(BaseDataType::T_I64, 1); + H5::DataType nativeTypeI64 = IO::HDF5::HDF5IO::getNativeType(typeI64); + REQUIRE(nativeTypeI64.getId() == H5::PredType::NATIVE_INT64.getId()); + + // Test for T_U8 + BaseDataType typeU8(BaseDataType::T_U8, 1); + H5::DataType nativeTypeU8 = IO::HDF5::HDF5IO::getNativeType(typeU8); + REQUIRE(nativeTypeU8.getId() == H5::PredType::NATIVE_UINT8.getId()); + + // Test for T_U16 + BaseDataType typeU16(BaseDataType::T_U16, 1); + H5::DataType nativeTypeU16 = IO::HDF5::HDF5IO::getNativeType(typeU16); + REQUIRE(nativeTypeU16.getId() == H5::PredType::NATIVE_UINT16.getId()); + + // Test for T_U32 + BaseDataType typeU32(BaseDataType::T_U32, 1); + H5::DataType nativeTypeU32 = IO::HDF5::HDF5IO::getNativeType(typeU32); + REQUIRE(nativeTypeU32.getId() == H5::PredType::NATIVE_UINT32.getId()); + + // Test for T_U64 + BaseDataType typeU64(BaseDataType::T_U64, 1); + H5::DataType nativeTypeU64 = IO::HDF5::HDF5IO::getNativeType(typeU64); + REQUIRE(nativeTypeU64.getId() == H5::PredType::NATIVE_UINT64.getId()); + } + + SECTION("Floating Point Types") + { + // Test for T_F32 + BaseDataType typeF32(BaseDataType::T_F32, 1); + H5::DataType nativeTypeF32 = IO::HDF5::HDF5IO::getNativeType(typeF32); + REQUIRE(nativeTypeF32.getId() == H5::PredType::NATIVE_FLOAT.getId()); + + // Test for T_F64 + BaseDataType typeF64(BaseDataType::T_F64, 1); + H5::DataType nativeTypeF64 = IO::HDF5::HDF5IO::getNativeType(typeF64); + REQUIRE(nativeTypeF64.getId() == H5::PredType::NATIVE_DOUBLE.getId()); + } + + SECTION("String Types") + { + // Test for T_STR + BaseDataType typeSTR(BaseDataType::T_STR, 256); + H5::DataType nativeTypeSTR = IO::HDF5::HDF5IO::getNativeType(typeSTR); + REQUIRE(nativeTypeSTR.getId() == H5::StrType(H5::PredType::C_S1, +256).getId()); + + // Test for V_STR + BaseDataType typeVSTR(BaseDataType::V_STR, 1); + H5::DataType nativeTypeVSTR = IO::HDF5::HDF5IO::getNativeType(typeVSTR); + REQUIRE(nativeTypeVSTR.getId() == H5::StrType(H5::PredType::C_S1, +H5T_VARIABLE).getId()); + } + + SECTION("Array Types") + { + // Test for array types + BaseDataType typeArray(BaseDataType::T_I32, 5); + H5::DataType nativeTypeArray = IO::HDF5::HDF5IO::getNativeType(typeArray); + hsize_t size = 5; + H5::ArrayType expectedArrayType(H5::PredType::NATIVE_INT32, 1, &size); + REQUIRE(nativeTypeArray.getId() == expectedArrayType.getId()); + } +} */ \ No newline at end of file From 57aeebb2fc7a965da5f5056a0f58bb6ec3988eef Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 25 Dec 2024 17:19:03 -0800 Subject: [PATCH 03/19] Add unit tests for HDF5IO::getNativeType --- src/io/hdf5/HDF5IO.cpp | 34 +++++++++++++++++----------------- tests/testHDF5IO.cpp | 40 ++++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index 4c638ca..1600829 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -905,49 +905,49 @@ H5::DataType HDF5IO::getNativeType(IO::BaseDataType type) switch (type.type) { case IO::BaseDataType::Type::T_I8: - baseType = PredType::NATIVE_INT8; + baseType = H5::PredType::NATIVE_INT8; break; case IO::BaseDataType::Type::T_I16: - baseType = PredType::NATIVE_INT16; + baseType = H5::PredType::NATIVE_INT16; break; case IO::BaseDataType::Type::T_I32: - baseType = PredType::NATIVE_INT32; + baseType = H5::PredType::NATIVE_INT32; break; case IO::BaseDataType::Type::T_I64: - baseType = PredType::NATIVE_INT64; + baseType = H5::PredType::NATIVE_INT64; break; case IO::BaseDataType::Type::T_U8: - baseType = PredType::NATIVE_UINT8; + baseType = H5::PredType::NATIVE_UINT8; break; case IO::BaseDataType::Type::T_U16: - baseType = PredType::NATIVE_UINT16; + baseType = H5::PredType::NATIVE_UINT16; break; case IO::BaseDataType::Type::T_U32: - baseType = PredType::NATIVE_UINT32; + baseType = H5::PredType::NATIVE_UINT32; break; case IO::BaseDataType::Type::T_U64: - baseType = PredType::NATIVE_UINT64; + baseType = H5::PredType::NATIVE_UINT64; break; case IO::BaseDataType::Type::T_F32: - baseType = PredType::NATIVE_FLOAT; + baseType = H5::PredType::NATIVE_FLOAT; break; case IO::BaseDataType::Type::T_F64: - baseType = PredType::NATIVE_DOUBLE; + baseType = H5::PredType::NATIVE_DOUBLE; break; case IO::BaseDataType::Type::T_STR: - return StrType(PredType::C_S1, type.typeSize); - break; + return H5::StrType(H5::PredType::C_S1, type.typeSize); case IO::BaseDataType::Type::V_STR: - return StrType(PredType::C_S1, H5T_VARIABLE); - break; + return H5::StrType(H5::PredType::C_S1, H5T_VARIABLE); default: - baseType = PredType::NATIVE_INT32; + baseType = H5::PredType::NATIVE_INT32; } + if (type.typeSize > 1) { hsize_t size = type.typeSize; - return ArrayType(baseType, 1, &size); - } else + return H5::ArrayType(baseType, 1, &size); + } else { return baseType; + } } H5::DataType HDF5IO::getH5Type(IO::BaseDataType type) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 2881886..1554943 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -541,47 +541,47 @@ TEST_CASE("getH5ObjectType", "[hdf5io]") hdf5io.close(); } -/* TEST_CASE("HDF5IO::getNativeType", "[hdf5io]") +TEST_CASE("HDF5IO::getNativeType", "[hdf5io]") { SECTION("Integer Types") { // Test for T_I8 - BaseDataType typeI8(BaseDataType::T_I8, 1); + IO::BaseDataType typeI8(IO::BaseDataType::T_I8, 1); H5::DataType nativeTypeI8 = IO::HDF5::HDF5IO::getNativeType(typeI8); REQUIRE(nativeTypeI8.getId() == H5::PredType::NATIVE_INT8.getId()); // Test for T_I16 - BaseDataType typeI16(BaseDataType::T_I16, 1); + IO::BaseDataType typeI16(IO::BaseDataType::T_I16, 1); H5::DataType nativeTypeI16 = IO::HDF5::HDF5IO::getNativeType(typeI16); REQUIRE(nativeTypeI16.getId() == H5::PredType::NATIVE_INT16.getId()); // Test for T_I32 - BaseDataType typeI32(BaseDataType::T_I32, 1); + IO::BaseDataType typeI32(IO::BaseDataType::T_I32, 1); H5::DataType nativeTypeI32 = IO::HDF5::HDF5IO::getNativeType(typeI32); REQUIRE(nativeTypeI32.getId() == H5::PredType::NATIVE_INT32.getId()); // Test for T_I64 - BaseDataType typeI64(BaseDataType::T_I64, 1); + IO::BaseDataType typeI64(IO::BaseDataType::T_I64, 1); H5::DataType nativeTypeI64 = IO::HDF5::HDF5IO::getNativeType(typeI64); REQUIRE(nativeTypeI64.getId() == H5::PredType::NATIVE_INT64.getId()); // Test for T_U8 - BaseDataType typeU8(BaseDataType::T_U8, 1); + IO::BaseDataType typeU8(IO::BaseDataType::T_U8, 1); H5::DataType nativeTypeU8 = IO::HDF5::HDF5IO::getNativeType(typeU8); REQUIRE(nativeTypeU8.getId() == H5::PredType::NATIVE_UINT8.getId()); // Test for T_U16 - BaseDataType typeU16(BaseDataType::T_U16, 1); + IO::BaseDataType typeU16(IO::BaseDataType::T_U16, 1); H5::DataType nativeTypeU16 = IO::HDF5::HDF5IO::getNativeType(typeU16); REQUIRE(nativeTypeU16.getId() == H5::PredType::NATIVE_UINT16.getId()); // Test for T_U32 - BaseDataType typeU32(BaseDataType::T_U32, 1); + IO::BaseDataType typeU32(IO::BaseDataType::T_U32, 1); H5::DataType nativeTypeU32 = IO::HDF5::HDF5IO::getNativeType(typeU32); REQUIRE(nativeTypeU32.getId() == H5::PredType::NATIVE_UINT32.getId()); // Test for T_U64 - BaseDataType typeU64(BaseDataType::T_U64, 1); + IO::BaseDataType typeU64(IO::BaseDataType::T_U64, 1); H5::DataType nativeTypeU64 = IO::HDF5::HDF5IO::getNativeType(typeU64); REQUIRE(nativeTypeU64.getId() == H5::PredType::NATIVE_UINT64.getId()); } @@ -589,12 +589,12 @@ TEST_CASE("getH5ObjectType", "[hdf5io]") SECTION("Floating Point Types") { // Test for T_F32 - BaseDataType typeF32(BaseDataType::T_F32, 1); + IO::BaseDataType typeF32(IO::BaseDataType::T_F32, 1); H5::DataType nativeTypeF32 = IO::HDF5::HDF5IO::getNativeType(typeF32); REQUIRE(nativeTypeF32.getId() == H5::PredType::NATIVE_FLOAT.getId()); // Test for T_F64 - BaseDataType typeF64(BaseDataType::T_F64, 1); + IO::BaseDataType typeF64(IO::BaseDataType::T_F64, 1); H5::DataType nativeTypeF64 = IO::HDF5::HDF5IO::getNativeType(typeF64); REQUIRE(nativeTypeF64.getId() == H5::PredType::NATIVE_DOUBLE.getId()); } @@ -602,25 +602,25 @@ TEST_CASE("getH5ObjectType", "[hdf5io]") SECTION("String Types") { // Test for T_STR - BaseDataType typeSTR(BaseDataType::T_STR, 256); + IO::BaseDataType typeSTR(IO::BaseDataType::T_STR, 256); H5::DataType nativeTypeSTR = IO::HDF5::HDF5IO::getNativeType(typeSTR); - REQUIRE(nativeTypeSTR.getId() == H5::StrType(H5::PredType::C_S1, -256).getId()); + REQUIRE(nativeTypeSTR.getSize() + == H5::StrType(H5::PredType::C_S1, 256).getSize()); // Test for V_STR - BaseDataType typeVSTR(BaseDataType::V_STR, 1); + IO::BaseDataType typeVSTR(IO::BaseDataType::V_STR, 1); H5::DataType nativeTypeVSTR = IO::HDF5::HDF5IO::getNativeType(typeVSTR); - REQUIRE(nativeTypeVSTR.getId() == H5::StrType(H5::PredType::C_S1, -H5T_VARIABLE).getId()); + REQUIRE(nativeTypeVSTR.getSize() + == H5::StrType(H5::PredType::C_S1, H5T_VARIABLE).getSize()); } SECTION("Array Types") { // Test for array types - BaseDataType typeArray(BaseDataType::T_I32, 5); + IO::BaseDataType typeArray(IO::BaseDataType::T_I32, 5); H5::DataType nativeTypeArray = IO::HDF5::HDF5IO::getNativeType(typeArray); hsize_t size = 5; H5::ArrayType expectedArrayType(H5::PredType::NATIVE_INT32, 1, &size); - REQUIRE(nativeTypeArray.getId() == expectedArrayType.getId()); + REQUIRE(nativeTypeArray.getSize() == expectedArrayType.getSize()); } -} */ \ No newline at end of file +} \ No newline at end of file From d7f883a8409374baa059c525986e7e5adb06fbeb Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Wed, 25 Dec 2024 17:29:27 -0800 Subject: [PATCH 04/19] Add unit test for HDF5IO::getH5Type --- src/io/hdf5/HDF5IO.hpp | 9 +++++ tests/testHDF5IO.cpp | 84 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/src/io/hdf5/HDF5IO.hpp b/src/io/hdf5/HDF5IO.hpp index ad026ce..e1b715a 100644 --- a/src/io/hdf5/HDF5IO.hpp +++ b/src/io/hdf5/HDF5IO.hpp @@ -318,6 +318,11 @@ class HDF5IO : public BaseIO /** * @brief Returns the HDF5 native data type for a given base data type. + * + * Native types are platform-dependent and represent the data types as they + * are stored in the memory of the machine where the HDF5 file is created or + * read. + * * @param type The base data type. * @return The HDF5 native data type. */ @@ -325,6 +330,10 @@ class HDF5IO : public BaseIO /** * @brief Returns the HDF5 data type for a given base data type. + * + * Standard types are platform-independent and represent the data types + * in a consistent format, regardless of the machine architecture. + * * @param type The base data type. * @return The HDF5 data type. */ diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 1554943..e06ef56 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -623,4 +623,88 @@ TEST_CASE("HDF5IO::getNativeType", "[hdf5io]") H5::ArrayType expectedArrayType(H5::PredType::NATIVE_INT32, 1, &size); REQUIRE(nativeTypeArray.getSize() == expectedArrayType.getSize()); } +} + +TEST_CASE("HDF5IO::getH5Type", "[hdf5io]") +{ + SECTION("Integer Types") + { + // Test for T_I8 + IO::BaseDataType typeI8(IO::BaseDataType::T_I8, 1); + H5::DataType h5TypeI8 = IO::HDF5::HDF5IO::getH5Type(typeI8); + REQUIRE(h5TypeI8.getSize() == H5::PredType::STD_I8LE.getSize()); + + // Test for T_I16 + IO::BaseDataType typeI16(IO::BaseDataType::T_I16, 1); + H5::DataType h5TypeI16 = IO::HDF5::HDF5IO::getH5Type(typeI16); + REQUIRE(h5TypeI16.getSize() == H5::PredType::STD_I16LE.getSize()); + + // Test for T_I32 + IO::BaseDataType typeI32(IO::BaseDataType::T_I32, 1); + H5::DataType h5TypeI32 = IO::HDF5::HDF5IO::getH5Type(typeI32); + REQUIRE(h5TypeI32.getSize() == H5::PredType::STD_I32LE.getSize()); + + // Test for T_I64 + IO::BaseDataType typeI64(IO::BaseDataType::T_I64, 1); + H5::DataType h5TypeI64 = IO::HDF5::HDF5IO::getH5Type(typeI64); + REQUIRE(h5TypeI64.getSize() == H5::PredType::STD_I64LE.getSize()); + + // Test for T_U8 + IO::BaseDataType typeU8(IO::BaseDataType::T_U8, 1); + H5::DataType h5TypeU8 = IO::HDF5::HDF5IO::getH5Type(typeU8); + REQUIRE(h5TypeU8.getSize() == H5::PredType::STD_U8LE.getSize()); + + // Test for T_U16 + IO::BaseDataType typeU16(IO::BaseDataType::T_U16, 1); + H5::DataType h5TypeU16 = IO::HDF5::HDF5IO::getH5Type(typeU16); + REQUIRE(h5TypeU16.getSize() == H5::PredType::STD_U16LE.getSize()); + + // Test for T_U32 + IO::BaseDataType typeU32(IO::BaseDataType::T_U32, 1); + H5::DataType h5TypeU32 = IO::HDF5::HDF5IO::getH5Type(typeU32); + REQUIRE(h5TypeU32.getSize() == H5::PredType::STD_U32LE.getSize()); + + // Test for T_U64 + IO::BaseDataType typeU64(IO::BaseDataType::T_U64, 1); + H5::DataType h5TypeU64 = IO::HDF5::HDF5IO::getH5Type(typeU64); + REQUIRE(h5TypeU64.getSize() == H5::PredType::STD_U64LE.getSize()); + } + + SECTION("Floating Point Types") + { + // Test for T_F32 + IO::BaseDataType typeF32(IO::BaseDataType::T_F32, 1); + H5::DataType h5TypeF32 = IO::HDF5::HDF5IO::getH5Type(typeF32); + REQUIRE(h5TypeF32.getSize() == H5::PredType::IEEE_F32LE.getSize()); + + // Test for T_F64 + IO::BaseDataType typeF64(IO::BaseDataType::T_F64, 1); + H5::DataType h5TypeF64 = IO::HDF5::HDF5IO::getH5Type(typeF64); + REQUIRE(h5TypeF64.getSize() == H5::PredType::IEEE_F64LE.getSize()); + } + + SECTION("String Types") + { + // Test for T_STR + IO::BaseDataType typeSTR(IO::BaseDataType::T_STR, 256); + H5::DataType h5TypeSTR = IO::HDF5::HDF5IO::getH5Type(typeSTR); + REQUIRE(h5TypeSTR.getSize() + == H5::StrType(H5::PredType::C_S1, 256).getSize()); + + // Test for V_STR + IO::BaseDataType typeVSTR(IO::BaseDataType::V_STR, 1); + H5::DataType h5TypeVSTR = IO::HDF5::HDF5IO::getH5Type(typeVSTR); + REQUIRE(h5TypeVSTR.getSize() + == H5::StrType(H5::PredType::C_S1, H5T_VARIABLE).getSize()); + } + + SECTION("Array Types") + { + // Test for array types + IO::BaseDataType typeArray(IO::BaseDataType::T_I32, 5); + H5::DataType h5TypeArray = IO::HDF5::HDF5IO::getH5Type(typeArray); + hsize_t size = 5; + H5::ArrayType expectedArrayType(H5::PredType::STD_I32LE, 1, &size); + REQUIRE(h5TypeArray.getSize() == expectedArrayType.getSize()); + } } \ No newline at end of file From 8d08722d484d94384a403c6ef13d60a3542807d0 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 13:33:14 -0800 Subject: [PATCH 05/19] Add unit test for HDF5IO::objectExists --- tests/testHDF5IO.cpp | 66 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 7 deletions(-) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index e06ef56..3b815a4 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -29,7 +29,7 @@ std::string executablePath = "./reader_executable"; using namespace AQNWB; namespace fs = std::filesystem; -TEST_CASE("writeGroup", "[hdf5io]") +TEST_CASE("HDF5IO::createGroup", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("testGroup.h5"); @@ -43,7 +43,7 @@ TEST_CASE("writeGroup", "[hdf5io]") } } -TEST_CASE("searchGroup", "[hdf5io]") +TEST_CASE("HDF5IO::getGroupObjects", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("searchGroup.h5"); @@ -63,7 +63,7 @@ TEST_CASE("searchGroup", "[hdf5io]") hdf5io.close(); } -TEST_CASE("writeDataset", "[hdf5io]") +TEST_CASE("HDF5IO; write datasets", "[hdf5io]") { std::vector testData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -259,7 +259,7 @@ TEST_CASE("writeDataset", "[hdf5io]") } } -TEST_CASE("writeAttributes", "[hdf5io]") +TEST_CASE("HDF5IO; create attributes", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("test_attributes.h5"); @@ -313,7 +313,7 @@ TEST_CASE("writeAttributes", "[hdf5io]") hdf5io.close(); } -TEST_CASE("SWMRmode", "[hdf5io]") +TEST_CASE("HDF5IO; SWMR mode", "[hdf5io]") { SECTION("useSWMRMODE") { @@ -463,7 +463,7 @@ TEST_CASE("SWMRmode", "[hdf5io]") } } -TEST_CASE("readDataset", "[hdf5io]") +TEST_CASE("HDF5IO; read dataset", "[hdf5io]") { std::vector testData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -510,7 +510,7 @@ TEST_CASE("readDataset", "[hdf5io]") } } -TEST_CASE("getH5ObjectType", "[hdf5io]") +TEST_CASE("HDF5IO::getH5ObjectType", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("test_getH5ObjectType.h5"); @@ -707,4 +707,56 @@ TEST_CASE("HDF5IO::getH5Type", "[hdf5io]") H5::ArrayType expectedArrayType(H5::PredType::STD_I32LE, 1, &size); REQUIRE(h5TypeArray.getSize() == expectedArrayType.getSize()); } +} + +TEST_CASE("HDF5IO::objectExists", "[hdf5io]") +{ + // create and open file + std::string filename = getTestFilePath("test_objectExists.h5"); + IO::HDF5::HDF5IO hdf5io(filename); + hdf5io.open(); + + SECTION("existing group") + { + hdf5io.createGroup("/existingGroup"); + REQUIRE(hdf5io.objectExists("/existingGroup") == true); + } + + SECTION("non-existing group") + { + REQUIRE(hdf5io.objectExists("/nonExistingGroup") == false); + } + + SECTION("existing dataset") + { + std::vector testData = {1, 2, 3, 4, 5}; + std::string dataPath = "/existingDataset"; + hdf5io.createArrayDataSet( + BaseDataType::I32, SizeArray {0}, SizeArray {1}, dataPath); + REQUIRE(hdf5io.objectExists(dataPath) == true); + } + + SECTION("non-existing dataset") + { + REQUIRE(hdf5io.objectExists("/nonExistingDataset") == false); + } + + SECTION("existing attribute") + { + hdf5io.createGroup("/groupWithAttribute"); + const int data = 1; + hdf5io.createAttribute( + BaseDataType::I32, &data, "/groupWithAttribute", "existingAttribute"); + REQUIRE(hdf5io.attributeExists("/groupWithAttribute/existingAttribute") + == true); + } + + SECTION("non-existing attribute") + { + REQUIRE(hdf5io.attributeExists("/groupWithAttribute/nonExistingAttribute") + == false); + } + + // close file + hdf5io.close(); } \ No newline at end of file From f56ea7249e2610846aac2ca372cbc3a1d167e57f Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 13:39:09 -0800 Subject: [PATCH 06/19] Add unit tests for HDF5IO::attributeExists --- tests/testHDF5IO.cpp | 49 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 3b815a4..2716ecf 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -757,6 +757,55 @@ TEST_CASE("HDF5IO::objectExists", "[hdf5io]") == false); } + // close file + hdf5io.close(); +} + +TEST_CASE("HDF5IOI::attributeExists", "[hdf5io]") +{ + // create and open file + std::string filename = getTestFilePath("test_attributeExists.h5"); + IO::HDF5::HDF5IO hdf5io(filename); + hdf5io.open(); + + hdf5io.createGroup("/data"); + + SECTION("existing attribute") + { + const int data = 1; + hdf5io.createAttribute( + BaseDataType::I32, &data, "/data", "existingAttribute"); + REQUIRE(hdf5io.attributeExists("/data/existingAttribute") == true); + } + + SECTION("non-existing attribute") + { + REQUIRE(hdf5io.attributeExists("/data/nonExistingAttribute") == false); + } + + SECTION("existing string attribute") + { + const std::string data = "test_string"; + hdf5io.createAttribute(data, "/data", "existingStringAttribute"); + REQUIRE(hdf5io.attributeExists("/data/existingStringAttribute") == true); + } + + SECTION("existing string array attribute") + { + const std::vector data = {"str1", "str2", "str3"}; + hdf5io.createAttribute(data, "/data", "existingStringArrayAttribute"); + REQUIRE(hdf5io.attributeExists("/data/existingStringArrayAttribute") + == true); + } + + SECTION("existing reference attribute") + { + hdf5io.createGroup("/referenceTarget"); + hdf5io.createReferenceAttribute( + "/referenceTarget", "/data", "existingReferenceAttribute"); + REQUIRE(hdf5io.attributeExists("/data/existingReferenceAttribute") == true); + } + // close file hdf5io.close(); } \ No newline at end of file From 2d72e30dafc757e336cf24bceb5aaec39c0fd7e0 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 13:48:48 -0800 Subject: [PATCH 07/19] add unit test for HDF5IO::getGroupObjects --- tests/testHDF5IO.cpp | 55 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 2716ecf..6fb4d63 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -46,20 +46,53 @@ TEST_CASE("HDF5IO::createGroup", "[hdf5io]") TEST_CASE("HDF5IO::getGroupObjects", "[hdf5io]") { // create and open file - std::string filename = getTestFilePath("searchGroup.h5"); + std::string filename = getTestFilePath("test_getGroupObjects.h5"); IO::HDF5::HDF5IO hdf5io(filename); hdf5io.open(); - hdf5io.createGroup("/data"); - hdf5io.createGroup("/data/test"); - hdf5io.createArrayDataSet( - BaseDataType::I32, SizeArray {0}, SizeArray {1}, "/data/mydata"); - hdf5io.flush(); - auto group_content = hdf5io.getGroupObjects("/data"); - REQUIRE(group_content.size() == 2); - auto group_content2 = hdf5io.getGroupObjects("/"); - REQUIRE(group_content2.size() == 1); - REQUIRE(group_content2[0] == "data"); + SECTION("empty group") + { + hdf5io.createGroup("/emptyGroup"); + auto groupContent = hdf5io.getGroupObjects("/emptyGroup"); + REQUIRE(groupContent.size() == 0); + } + + SECTION("group with datasets and subgroups") + { + hdf5io.createGroup("/data"); + hdf5io.createGroup("/data/subgroup1"); + hdf5io.createGroup("/data/subgroup2"); + hdf5io.createArrayDataSet( + BaseDataType::I32, SizeArray {0}, SizeArray {1}, "/data/dataset1"); + hdf5io.createArrayDataSet( + BaseDataType::I32, SizeArray {0}, SizeArray {1}, "/data/dataset2"); + + auto groupContent = hdf5io.getGroupObjects("/data"); + REQUIRE(groupContent.size() == 4); + REQUIRE(std::find(groupContent.begin(), groupContent.end(), "subgroup1") + != groupContent.end()); + REQUIRE(std::find(groupContent.begin(), groupContent.end(), "subgroup2") + != groupContent.end()); + REQUIRE(std::find(groupContent.begin(), groupContent.end(), "dataset1") + != groupContent.end()); + REQUIRE(std::find(groupContent.begin(), groupContent.end(), "dataset2") + != groupContent.end()); + } + + SECTION("root group") + { + hdf5io.createGroup("/rootGroup1"); + hdf5io.createGroup("/rootGroup2"); + + auto groupContent = hdf5io.getGroupObjects("/"); + REQUIRE(groupContent.size() == 2); + REQUIRE(std::find(groupContent.begin(), groupContent.end(), "rootGroup1") + != groupContent.end()); + REQUIRE(std::find(groupContent.begin(), groupContent.end(), "rootGroup2") + != groupContent.end()); + } + + // close file hdf5io.close(); } From fa71fbd8e33f5211e3af7b2da85f2e1b0265c31a Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 13:54:56 -0800 Subject: [PATCH 08/19] Add unit test for HDF5IO::getStorageObjectType --- tests/testHDF5IO.cpp | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 6fb4d63..0de57ed 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -839,6 +839,65 @@ TEST_CASE("HDF5IOI::attributeExists", "[hdf5io]") REQUIRE(hdf5io.attributeExists("/data/existingReferenceAttribute") == true); } + // close file + hdf5io.close(); +} + +TEST_CASE("getStorageObjectType", "[hdf5io]") +{ + // create and open file + std::string filename = getTestFilePath("test_getStorageObjectType.h5"); + IO::HDF5::HDF5IO hdf5io(filename); + hdf5io.open(); + + SECTION("group") + { + hdf5io.createGroup("/testGroup"); + REQUIRE(hdf5io.getStorageObjectType("/testGroup") + == StorageObjectType::Group); + } + + SECTION("dataset") + { + hdf5io.createArrayDataSet( + BaseDataType::I32, SizeArray {0}, SizeArray {1}, "/testDataset"); + REQUIRE(hdf5io.getStorageObjectType("/testDataset") + == StorageObjectType::Dataset); + } + + SECTION("attribute") + { + hdf5io.createGroup("/groupWithAttribute"); + const int data = 1; + hdf5io.createAttribute( + BaseDataType::I32, &data, "/groupWithAttribute", "testAttribute"); + REQUIRE(hdf5io.getStorageObjectType("/groupWithAttribute/testAttribute") + == StorageObjectType::Attribute); + } + + SECTION("non-existing object") + { + REQUIRE(hdf5io.getStorageObjectType("/nonExistingObject") + == StorageObjectType::Undefined); + } + + SECTION("link to group") + { + hdf5io.createGroup("/originalGroup"); + hdf5io.createLink("/linkToGroup", "/originalGroup"); + REQUIRE(hdf5io.getStorageObjectType("/linkToGroup") + == StorageObjectType::Group); + } + + SECTION("link to dataset") + { + hdf5io.createArrayDataSet( + BaseDataType::I32, SizeArray {0}, SizeArray {1}, "/originalDataset"); + hdf5io.createLink("/linkToDataset", "/originalDataset"); + REQUIRE(hdf5io.getStorageObjectType("/linkToDataset") + == StorageObjectType::Dataset); + } + // close file hdf5io.close(); } \ No newline at end of file From 05d33edf57a33399bfa791f0c7049b130ddee48b Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 14:39:15 -0800 Subject: [PATCH 09/19] Fix reading of arrays and select dtypes for HDF5IO::readAttribute --- src/io/hdf5/HDF5IO.cpp | 44 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index 1600829..b9f84ef 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -290,6 +290,12 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute( } else if (dataType == H5::PredType::NATIVE_FLOAT) { result.data = readDataHelper(attribute, numElements); result.typeIndex = typeid(float); + } else if (dataType == H5::PredType::NATIVE_INT32) { + result.data = readDataHelper(attribute, numElements); + result.typeIndex = typeid(int32_t); + } else if (dataType == H5::PredType::NATIVE_UINT32) { + result.data = readDataHelper(attribute, numElements); + result.typeIndex = typeid(uint32_t); } else if (dataType == H5::PredType::NATIVE_INT) { result.data = readDataHelper(attribute, numElements); result.typeIndex = typeid(int); @@ -308,6 +314,44 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute( } else if (dataType == H5::PredType::NATIVE_ULLONG) { result.data = readDataHelper(attribute, numElements); result.typeIndex = typeid(unsigned long long); + } else if (dataType.getClass() == H5T_ARRAY) { + // Handle array attributes + H5::ArrayType arrayType(dataType.getId()); + H5::DataType baseType = arrayType.getSuper(); + int arrayRank = arrayType.getArrayNDims(); + std::vector arrayDims(arrayRank); + arrayType.getArrayDims(arrayDims.data()); + + // Convert arrayDims to std::vector + std::vector convertedArrayDims(arrayDims.begin(), arrayDims.end()); + + // Update the shape to reflect the array dimensions + result.shape = convertedArrayDims; + + size_t arrayNumElements = 1; + for (const auto dim : arrayDims) { + arrayNumElements *= dim; + } + + if (baseType == H5::PredType::NATIVE_INT32) { + result.data = + readDataHelper(attribute, numElements * arrayNumElements); + result.typeIndex = typeid(int32_t); + } else if (baseType == H5::PredType::NATIVE_UINT32) { + result.data = + readDataHelper(attribute, numElements * arrayNumElements); + result.typeIndex = typeid(uint32_t); + } else if (baseType == H5::PredType::NATIVE_FLOAT) { + result.data = + readDataHelper(attribute, numElements * arrayNumElements); + result.typeIndex = typeid(float); + } else if (baseType == H5::PredType::NATIVE_DOUBLE) { + result.data = + readDataHelper(attribute, numElements * arrayNumElements); + result.typeIndex = typeid(double); + } else { + throw std::runtime_error("Unsupported array base data type"); + } } else { throw std::runtime_error("Unsupported data type"); } From 84e10a511555357b507e70e9a144165d57141edc Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 14:39:36 -0800 Subject: [PATCH 10/19] Add initial unit tests for HDF5IO::readAttribute --- tests/testHDF5IO.cpp | 108 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 0de57ed..4ada5aa 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include "Channel.hpp" @@ -843,7 +844,7 @@ TEST_CASE("HDF5IOI::attributeExists", "[hdf5io]") hdf5io.close(); } -TEST_CASE("getStorageObjectType", "[hdf5io]") +TEST_CASE("HDF5IO::getStorageObjectType", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("test_getStorageObjectType.h5"); @@ -898,6 +899,111 @@ TEST_CASE("getStorageObjectType", "[hdf5io]") == StorageObjectType::Dataset); } + // close file + hdf5io.close(); +} + +TEST_CASE("readAttribute", "[hdf5io]") +{ + // create and open file + std::string filename = getTestFilePath("test_readAttribute.h5"); + IO::HDF5::HDF5IO hdf5io(filename); + hdf5io.open(); + + hdf5io.createGroup("/data"); + + SECTION("read integer attribute") + { + const int32_t writeData = 42; + hdf5io.createAttribute( + BaseDataType::I32, &writeData, "/data", "intAttribute"); + auto readDataGeneric = hdf5io.readAttribute("/data/intAttribute"); + auto readData = IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.empty()); // Scalar attribute + REQUIRE(readData.data.size() == 1); + REQUIRE(readData.data[0] == writeData); + } + + SECTION("read float attribute") + { + const float writeData = 3.14f; + hdf5io.createAttribute( + BaseDataType::F32, &writeData, "/data", "floatAttribute"); + auto readDataGeneric = hdf5io.readAttribute("/data/floatAttribute"); + auto readData = IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.empty()); // Scalar attribute + REQUIRE(readData.data.size() == 1); + REQUIRE(readData.data[0] == Catch::Approx(writeData).epsilon(0.001)); + } + + SECTION("read string attribute") + { + const std::string writeData = "test_string"; + hdf5io.createAttribute(writeData, "/data", "stringAttribute"); + auto readDataGeneric = hdf5io.readAttribute("/data/stringAttribute"); + auto readData = IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.empty()); // Scalar attribute + REQUIRE(readData.data.size() == 1); + REQUIRE(readData.data[0] == writeData); + } + + SECTION("read integer array attribute") + { + const int32_t writeData[] = {1, 2, 3, 4, 5}; + const int dataSize = sizeof(writeData) / sizeof(writeData[0]); + hdf5io.createAttribute( + BaseDataType::I32, writeData, "/data", "intArrayAttribute", dataSize); + auto readDataGeneric = hdf5io.readAttribute("/data/intArrayAttribute"); + auto readData = IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.size() == 1); // 1D array attribute + REQUIRE(readData.shape[0] == dataSize); + REQUIRE(readData.data.size() == dataSize); + for (int i = 0; i < dataSize; ++i) { + REQUIRE(readData.data[i] == writeData[i]); + } + } + + // TODO: read string array attribute currently fails with fatal error + /* + SECTION("read string array attribute") + { + const std::vector writeData = {"str1", "str2", "str3"}; + hdf5io.createAttribute(writeData, "/data", "stringArrayAttribute"); + auto readDataGeneric = + hdf5io.readAttribute("/data/stringArrayAttribute"); auto readData = + IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.size() == 1); // 1D array attribute + REQUIRE(readData.shape[0] == writeData.size()); + REQUIRE(readData.data.size() == writeData.size()); + for (size_t i = 0; i < writeData.size(); ++i) + { + REQUIRE(readData.data[i] == writeData[i]); + } + } + */ + + // TODO: read reference attribute test case is incomplete and does not compile + // yet + /* + SECTION("read reference attribute") + { + hdf5io.createGroup("/referenceTarget"); + hdf5io.createReferenceAttribute("/referenceTarget", "/data", + "referenceAttribute"); auto readDataGeneric = + hdf5io.readAttribute("/data/referenceAttribute"); auto readData = + IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.empty()); // Scalar attribute + REQUIRE(readData.data.size() == 1); + // You may need to add additional checks here to verify the reference + is correct + } + */ // close file hdf5io.close(); } \ No newline at end of file From 3b1feb7e24a688c955de371c889c13f4b7821e9b Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 15:11:03 -0800 Subject: [PATCH 11/19] Fix reading of string array attributes via HDF5IO::readAttribute --- src/io/hdf5/HDF5IO.cpp | 117 +++++++++++++++++++++++++---------------- tests/testHDF5IO.cpp | 33 +++++------- 2 files changed, 85 insertions(+), 65 deletions(-) diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index b9f84ef..0155df2 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -194,20 +194,63 @@ std::vector HDF5IO::readStringDataHelper( const H5::DataSpace& memspace, const H5::DataSpace& dataspace) const { + // Vector to store the read string data std::vector data(numElements); - if (const H5::DataSet* dataset = - dynamic_cast(&dataSource)) - { - H5::StrType strType(H5::PredType::C_S1); - dataset->read(data.data(), strType, memspace, dataspace); - } else if (const H5::Attribute* attribute = + + try { + // Check if the data source is a DataSet + if (const H5::DataSet* dataset = + dynamic_cast(&dataSource)) + { + // Get the string type of the dataset + H5::StrType strType = dataset->getStrType(); + + if (strType.isVariableStr()) { + // Handle variable-length strings + std::vector buffer(numElements); + dataset->read(buffer.data(), strType, memspace, dataspace); + + // Convert char* to std::string and free allocated memory + for (size_t i = 0; i < numElements; ++i) { + data[i] = std::string(buffer[i]); + free(buffer[i]); // Free the memory allocated by HDF5 + } + } else { + // Handle fixed-length strings + dataset->read(data.data(), strType, memspace, dataspace); + } + } + // Check if the data source is an Attribute + else if (const H5::Attribute* attribute = dynamic_cast(&dataSource)) - { - H5::StrType strType(H5::PredType::C_S1); - attribute->read(strType, data.data()); - } else { - throw std::runtime_error("Unsupported data source type"); + { + // Get the string type of the attribute + H5::StrType strType = attribute->getStrType(); + + if (strType.isVariableStr()) { + // Handle variable-length strings + std::vector buffer(numElements); + attribute->read(strType, buffer.data()); + + // Convert char* to std::string and free allocated memory + for (size_t i = 0; i < numElements; ++i) { + data[i] = std::string(buffer[i]); + free(buffer[i]); // Free the memory allocated by HDF5 + } + } else { + // Handle fixed-length strings + attribute->read(strType, data.data()); + } + } else { + // Throw an error if the data source type is unsupported + throw std::runtime_error("Unsupported data source type"); + } + } catch (const H5::Exception& e) { + // Catch and rethrow HDF5 exceptions with a detailed message + throw std::runtime_error("Failed to read string data: " + + std::string(e.getDetailMsg())); } + return data; } @@ -215,6 +258,7 @@ template std::vector HDF5IO::readStringDataHelper( const HDF5TYPE& dataSource, size_t numElements) const { + // Call the main readStringDataHelper function with default DataSpaces return this->readStringDataHelper( dataSource, numElements, H5::DataSpace(), H5::DataSpace()); } @@ -226,7 +270,6 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute( AQNWB::IO::DataBlockGeneric result; // Read the attribute auto attributePtr = this->getAttribute(dataPath); - // Make sure dataPath points to an attribute if (attributePtr == nullptr) { throw std::invalid_argument("attributePtr is null"); } @@ -236,25 +279,17 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute( // Determine the shape of the attribute H5::DataSpace dataspace = attribute.getSpace(); int rank = dataspace.getSimpleExtentNdims(); - if (rank == 0) { - // Scalar attribute - result.shape.clear(); - } else { + result.shape.clear(); + if (rank > 0) { std::vector tempShape(rank); dataspace.getSimpleExtentDims(tempShape.data(), nullptr); - result.shape.clear(); - result.shape.reserve(rank); - for (const auto& v : tempShape) { - result.shape.push_back(v); - } + result.shape.assign(tempShape.begin(), tempShape.end()); } // Determine the size of the attribute from the shape - size_t numElements = 1; // Scalar (default) - if (result.shape.size() > 0) { // N-dimensional array - for (const auto v : result.shape) { - numElements *= v; - } + size_t numElements = 1; + for (const auto v : result.shape) { + numElements *= v; } // Read the attribute into a vector of the appropriate type @@ -262,26 +297,19 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute( if (dataType.isVariableStr()) { // Handle variable-length strings std::vector stringData; - try { - // Allocate a buffer to hold the variable-length string data - char* buffer; - attribute.read(dataType, &buffer); - - // Convert the buffer to std::string - stringData.emplace_back(buffer); - - // Free the memory allocated by HDF5 - H5Dvlen_reclaim( - dataType.getId(), dataspace.getId(), H5P_DEFAULT, &buffer); - } catch (const H5::Exception& e) { - throw std::runtime_error( - "Failed to read variable-length string attribute: " - + std::string(e.getDetailMsg())); + std::vector buffer(numElements); + attribute.read(dataType, buffer.data()); + + for (size_t i = 0; i < numElements; ++i) { + stringData.emplace_back(buffer[i]); + free(buffer[i]); // Free the memory allocated by HDF5 } result.data = stringData; result.typeIndex = typeid(std::string); } else { - result.data = readStringDataHelper(attribute, numElements); + // Handle fixed-length strings + result.data = readStringDataHelper( + attribute, numElements, H5::DataSpace(), dataspace); result.typeIndex = typeid(std::string); } } else if (dataType == H5::PredType::NATIVE_DOUBLE) { @@ -322,11 +350,8 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute( std::vector arrayDims(arrayRank); arrayType.getArrayDims(arrayDims.data()); - // Convert arrayDims to std::vector - std::vector convertedArrayDims(arrayDims.begin(), arrayDims.end()); - // Update the shape to reflect the array dimensions - result.shape = convertedArrayDims; + result.shape.assign(arrayDims.begin(), arrayDims.end()); size_t arrayNumElements = 1; for (const auto dim : arrayDims) { diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 4ada5aa..1bd5686 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -967,25 +967,20 @@ TEST_CASE("readAttribute", "[hdf5io]") } } - // TODO: read string array attribute currently fails with fatal error - /* - SECTION("read string array attribute") - { - const std::vector writeData = {"str1", "str2", "str3"}; - hdf5io.createAttribute(writeData, "/data", "stringArrayAttribute"); - auto readDataGeneric = - hdf5io.readAttribute("/data/stringArrayAttribute"); auto readData = - IO::DataBlock::fromGeneric(readDataGeneric); - - REQUIRE(readData.shape.size() == 1); // 1D array attribute - REQUIRE(readData.shape[0] == writeData.size()); - REQUIRE(readData.data.size() == writeData.size()); - for (size_t i = 0; i < writeData.size(); ++i) - { - REQUIRE(readData.data[i] == writeData[i]); - } - } - */ + SECTION("read string array attribute") + { + const std::vector writeData = {"str1", "str2", "str3"}; + hdf5io.createAttribute(writeData, "/data", "stringArrayAttribute"); + auto readDataGeneric = hdf5io.readAttribute("/data/stringArrayAttribute"); + auto readData = IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.size() == 1); // 1D array attribute + REQUIRE(readData.shape[0] == writeData.size()); + REQUIRE(readData.data.size() == writeData.size()); + for (size_t i = 0; i < writeData.size(); ++i) { + REQUIRE(readData.data[i] == writeData[i]); + } + } // TODO: read reference attribute test case is incomplete and does not compile // yet From fe2c63e466ab197fa9a17616dfbfbf78f99b3137 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 15:41:34 -0800 Subject: [PATCH 12/19] Fix reading of reference attributes --- src/io/hdf5/HDF5IO.cpp | 24 +++++++-------- tests/testHDF5IO.cpp | 69 +++++++++++++++++++++++++++--------------- 2 files changed, 56 insertions(+), 37 deletions(-) diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index 0155df2..54cb3ef 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -377,6 +377,12 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute( } else { throw std::runtime_error("Unsupported array base data type"); } + } else if (dataType.getClass() == H5T_REFERENCE) { + // Handle object references + std::vector refData(numElements); + attribute.read(dataType, refData.data()); + result.data = refData; + result.typeIndex = typeid(hobj_ref_t); } else { throw std::runtime_error("Unsupported data type"); } @@ -664,21 +670,13 @@ Status HDF5IO::createReferenceAttribute(const std::string& referencePath, attr = loc->createAttribute(name, H5::PredType::STD_REF_OBJ, attr_space); } - hobj_ref_t* rdata = new hobj_ref_t[sizeof(hobj_ref_t)]; + hobj_ref_t rdata; + m_file->reference(&rdata, referencePath.c_str()); - m_file->reference(rdata, referencePath.c_str()); - - attr.write(H5::PredType::STD_REF_OBJ, rdata); - delete[] rdata; - - } catch (GroupIException error) { - error.printErrorStack(); - } catch (AttributeIException error) { - error.printErrorStack(); - } catch (FileIException error) { - error.printErrorStack(); - } catch (DataSetIException error) { + attr.write(H5::PredType::STD_REF_OBJ, &rdata); + } catch (const H5::Exception& error) { error.printErrorStack(); + return Status::Failure; } return Status::Success; diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 1bd5686..ea336b7 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -30,7 +31,7 @@ std::string executablePath = "./reader_executable"; using namespace AQNWB; namespace fs = std::filesystem; -TEST_CASE("HDF5IO::createGroup", "[hdf5io]") +TEST_CASE("createGroup", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("testGroup.h5"); @@ -44,7 +45,7 @@ TEST_CASE("HDF5IO::createGroup", "[hdf5io]") } } -TEST_CASE("HDF5IO::getGroupObjects", "[hdf5io]") +TEST_CASE("getGroupObjects", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("test_getGroupObjects.h5"); @@ -544,7 +545,7 @@ TEST_CASE("HDF5IO; read dataset", "[hdf5io]") } } -TEST_CASE("HDF5IO::getH5ObjectType", "[hdf5io]") +TEST_CASE("getH5ObjectType", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("test_getH5ObjectType.h5"); @@ -575,7 +576,7 @@ TEST_CASE("HDF5IO::getH5ObjectType", "[hdf5io]") hdf5io.close(); } -TEST_CASE("HDF5IO::getNativeType", "[hdf5io]") +TEST_CASE("getNativeType", "[hdf5io]") { SECTION("Integer Types") { @@ -659,7 +660,7 @@ TEST_CASE("HDF5IO::getNativeType", "[hdf5io]") } } -TEST_CASE("HDF5IO::getH5Type", "[hdf5io]") +TEST_CASE("getH5Type", "[hdf5io]") { SECTION("Integer Types") { @@ -743,7 +744,7 @@ TEST_CASE("HDF5IO::getH5Type", "[hdf5io]") } } -TEST_CASE("HDF5IO::objectExists", "[hdf5io]") +TEST_CASE("objectExists", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("test_objectExists.h5"); @@ -844,7 +845,7 @@ TEST_CASE("HDF5IOI::attributeExists", "[hdf5io]") hdf5io.close(); } -TEST_CASE("HDF5IO::getStorageObjectType", "[hdf5io]") +TEST_CASE("getStorageObjectType", "[hdf5io]") { // create and open file std::string filename = getTestFilePath("test_getStorageObjectType.h5"); @@ -982,23 +983,43 @@ TEST_CASE("readAttribute", "[hdf5io]") } } - // TODO: read reference attribute test case is incomplete and does not compile - // yet - /* - SECTION("read reference attribute") - { - hdf5io.createGroup("/referenceTarget"); - hdf5io.createReferenceAttribute("/referenceTarget", "/data", - "referenceAttribute"); auto readDataGeneric = - hdf5io.readAttribute("/data/referenceAttribute"); auto readData = - IO::DataBlock::fromGeneric(readDataGeneric); - - REQUIRE(readData.shape.empty()); // Scalar attribute - REQUIRE(readData.data.size() == 1); - // You may need to add additional checks here to verify the reference - is correct - } - */ + SECTION("read reference attribute") + { + // Create a target group to reference + hdf5io.createGroup("/referenceTarget"); + + // Create a reference attribute pointing to the target group + hdf5io.createReferenceAttribute( + "/referenceTarget", "/data", "referenceAttribute"); + + // Read the reference attribute + auto readDataGeneric = hdf5io.readAttribute("/data/referenceAttribute"); + auto readData = IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.empty()); // Scalar attribute + REQUIRE(readData.data.size() == 1); + + // Verify the reference points to the correct target group + H5::H5File file(hdf5io.getFileName(), H5F_ACC_RDONLY); + hid_t file_id = file.getId(); + hid_t ref_group_id = + H5Rdereference2(file_id, H5P_DEFAULT, H5R_OBJECT, &readData.data[0]); + + // Check if the dereferenced ID is valid + REQUIRE(ref_group_id >= 0); + + H5::Group refGroup(ref_group_id); + H5::Group targetGroup(file.openGroup("/referenceTarget")); + + // Compare the paths to ensure the reference points to the correct group + std::string refPath = refGroup.getObjName(); + std::string targetPath = targetGroup.getObjName(); + REQUIRE(refPath == targetPath); + + // Close the dereferenced group ID + H5Gclose(ref_group_id); + } + // close file hdf5io.close(); } \ No newline at end of file From 2b263438abe4d711974b83753fe78cf9544d031a Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 16:02:10 -0800 Subject: [PATCH 13/19] Add HDF5IO::readAttribute unit test for invalid and double attribute --- src/io/hdf5/HDF5IO.cpp | 4 +++- tests/testHDF5IO.cpp | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index 54cb3ef..04037e4 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -271,8 +271,10 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute( // Read the attribute auto attributePtr = this->getAttribute(dataPath); if (attributePtr == nullptr) { - throw std::invalid_argument("attributePtr is null"); + throw std::invalid_argument( + "HDF5IO::readAttribute, attribute does not exist."); } + H5::Attribute& attribute = *attributePtr; H5::DataType dataType = attribute.getDataType(); diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index ea336b7..f751f40 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -968,6 +968,19 @@ TEST_CASE("readAttribute", "[hdf5io]") } } + SECTION("read double attribute") + { + const double writeData = 2.718; + hdf5io.createAttribute( + BaseDataType::F64, &writeData, "/data", "doubleAttribute"); + auto readDataGeneric = hdf5io.readAttribute("/data/doubleAttribute"); + auto readData = IO::DataBlock::fromGeneric(readDataGeneric); + + REQUIRE(readData.shape.empty()); // Scalar attribute + REQUIRE(readData.data.size() == 1); + REQUIRE(readData.data[0] == Catch::Approx(writeData).epsilon(0.001)); + } + SECTION("read string array attribute") { const std::vector writeData = {"str1", "str2", "str3"}; @@ -1020,6 +1033,18 @@ TEST_CASE("readAttribute", "[hdf5io]") H5Gclose(ref_group_id); } + SECTION("read non-existent attribute") + { + REQUIRE_THROWS_AS(hdf5io.readAttribute("/data/nonExistentAttribute"), + std::invalid_argument); + } + + SECTION("read attribute from invalid path") + { + REQUIRE_THROWS_AS(hdf5io.readAttribute("/invalidPath/attribute"), + std::invalid_argument); + } + // close file hdf5io.close(); } \ No newline at end of file From 85cdf23bed7735af3f81dc27aa9da6703b3b5937 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 16:10:17 -0800 Subject: [PATCH 14/19] Add HDF5IO::readAttriute unit test for reading attribute from a dataset --- tests/testHDF5IO.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index f751f40..c46a7e2 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -1033,6 +1033,33 @@ TEST_CASE("readAttribute", "[hdf5io]") H5Gclose(ref_group_id); } + // TODO: Does not compile + SECTION("read attribute from dataset") + { + // Define the dimensions and chunking for the dataset + SizeArray dims = {10}; + SizeArray chunking = {10}; + + // Create the dataset using createArrayDataSet + auto dataset = hdf5io.createArrayDataSet( + BaseDataType::I32, dims, chunking, "/data/dataset"); + + // Define and create the attribute + const int32_t writeData = 123; + hdf5io.createAttribute( + BaseDataType::I32, &writeData, "/data/dataset", "datasetAttribute"); + + // Read the attribute + auto readDataGeneric = + hdf5io.readAttribute("/data/dataset/datasetAttribute"); + auto readData = IO::DataBlock::fromGeneric(readDataGeneric); + + // Verify the attribute data + REQUIRE(readData.shape.empty()); // Scalar attribute + REQUIRE(readData.data.size() == 1); + REQUIRE(readData.data[0] == writeData); + } + SECTION("read non-existent attribute") { REQUIRE_THROWS_AS(hdf5io.readAttribute("/data/nonExistentAttribute"), From 549bc22e60a1f17c9fefe450d515bdcdcb92a922 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 17:12:05 -0800 Subject: [PATCH 15/19] Fix handling of non-existent file ReadOnly and ReadWrite mode --- src/io/BaseIO.hpp | 6 ++++++ src/io/hdf5/HDF5IO.cpp | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/io/BaseIO.hpp b/src/io/BaseIO.hpp index 189e99a..eaf66d4 100644 --- a/src/io/BaseIO.hpp +++ b/src/io/BaseIO.hpp @@ -116,11 +116,17 @@ enum class FileMode /** * @brief Opens the file with both read and write access. + * + * Note: This is similar to r+ mode, so the file will not be created if it + * does not exist. */ ReadWrite, /** * @brief Opens the file in read only mode. + * + * Note: This is similar to r+ mode, so the file will not be created if it + * does not exist. */ ReadOnly }; diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index 04037e4..34def54 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -40,8 +40,15 @@ Status HDF5IO::open(FileMode mode) { int accFlags = 0; - if (m_opened) + if (m_opened) { return Status::Failure; + } + + if (!std::filesystem::exists(getFileName())) { + if (mode == FileMode::ReadWrite || mode == FileMode::ReadOnly) { + return Status::Failure; + } + } FileAccPropList fapl = FileAccPropList::DEFAULT; H5Pset_libver_bounds(fapl.getId(), H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); From 236881c8c9a7008da84dbe4dea191e9ac341232d Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Fri, 27 Dec 2024 17:12:32 -0800 Subject: [PATCH 16/19] Add unit tests for HDF5IO::open for file modes --- tests/testHDF5IO.cpp | 86 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index c46a7e2..9f6843a 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -31,6 +31,92 @@ std::string executablePath = "./reader_executable"; using namespace AQNWB; namespace fs = std::filesystem; +TEST_CASE("open - hdf5 file modes", "[hdf5io]") +{ + const std::string fileName = getTestFilePath("test_open_modes.h5"); + + // Ensure the file does not exist before starting the tests + std::filesystem::remove(fileName); + + SECTION("Open file in Overwrite mode") + { + IO::HDF5::HDF5IO hdf5io(fileName); + REQUIRE(hdf5io.open(IO::FileMode::Overwrite) == Status::Success); + REQUIRE(hdf5io.isOpen()); + + // Verify file is created and opened in Overwrite mode + H5::H5File file(fileName, H5F_ACC_RDONLY); + REQUIRE(file.getId() >= 0); + + // Clean up + file.close(); + hdf5io.close(); + std::filesystem::remove(fileName); + } + + SECTION("Open file in ReadWrite mode") + { + // Create a file first + { + IO::HDF5::HDF5IO hdf5io(fileName); + REQUIRE(hdf5io.open(IO::FileMode::Overwrite) == Status::Success); + hdf5io.close(); + } + + // Open the existing file in ReadWrite mode + IO::HDF5::HDF5IO hdf5io(fileName); + REQUIRE(hdf5io.open(IO::FileMode::ReadWrite) == Status::Success); + REQUIRE(hdf5io.isOpen()); + + // Verify file is opened in ReadWrite mode + H5::H5File file(fileName, H5F_ACC_RDWR); + REQUIRE(file.getId() >= 0); + + // Clean up + file.close(); + hdf5io.close(); + std::filesystem::remove(fileName); + } + + SECTION("Open file in ReadOnly mode") + { + // Create a file first + { + IO::HDF5::HDF5IO hdf5io(fileName); + REQUIRE(hdf5io.open(IO::FileMode::Overwrite) == Status::Success); + hdf5io.close(); + } + + // Open the existing file in ReadOnly mode + IO::HDF5::HDF5IO hdf5io(fileName); + REQUIRE(hdf5io.open(IO::FileMode::ReadOnly) == Status::Success); + REQUIRE(hdf5io.isOpen()); + + // Verify file is opened in ReadOnly mode + H5::H5File file(fileName, H5F_ACC_RDONLY | H5F_ACC_SWMR_READ); + REQUIRE(file.getId() >= 0); + + // Clean up + file.close(); + hdf5io.close(); + std::filesystem::remove(fileName); + } + + SECTION("Open non-existent file in ReadWrite mode") + { + IO::HDF5::HDF5IO hdf5io(fileName); + REQUIRE(hdf5io.open(IO::FileMode::ReadWrite) == Status::Failure); + REQUIRE_FALSE(hdf5io.isOpen()); + } + + SECTION("Open non-existent file in ReadOnly mode") + { + IO::HDF5::HDF5IO hdf5io(fileName); + REQUIRE(hdf5io.open(IO::FileMode::ReadOnly) == Status::Failure); + REQUIRE_FALSE(hdf5io.isOpen()); + } +} + TEST_CASE("createGroup", "[hdf5io]") { // create and open file From 884e2b2297199cbbf0d01073900e0563fc9b6a19 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 28 Dec 2024 14:41:25 -0800 Subject: [PATCH 17/19] Add additonal unit tests for HDF5IO::readDataset --- tests/testHDF5IO.cpp | 262 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 213 insertions(+), 49 deletions(-) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 9f6843a..2fb6272 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -191,7 +191,7 @@ TEST_CASE("HDF5IO; write datasets", "[hdf5io]") SECTION("write 1D data block to 1D dataset") { // open file - std::string path = getTestFilePath("1DData1DDataset.h5"); + std::string path = getTestFilePath("test_read1DData1DDataset.h5"); std::unique_ptr hdf5io = std::make_unique(path); hdf5io->open(); @@ -584,53 +584,6 @@ TEST_CASE("HDF5IO; SWMR mode", "[hdf5io]") } } -TEST_CASE("HDF5IO; read dataset", "[hdf5io]") -{ - std::vector testData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - - SECTION("read 1D data block of a 1D dataset") - { - // open file - std::string path = getTestFilePath("Read1DData1DDataset.h5"); - std::shared_ptr hdf5io = - std::make_shared(path); - hdf5io->open(); - - // Set up test data - std::string dataPath = "/1DData1DDataset"; - SizeType numSamples = 10; - - // Create HDF5RecordingData object and dataset - std::unique_ptr dataset = hdf5io->createArrayDataSet( - BaseDataType::I32, SizeArray {0}, SizeArray {1}, dataPath); - - // Write data block - std::vector dataShape = {numSamples}; - std::vector positionOffset = {0}; - dataset->writeDataBlock( - dataShape, positionOffset, BaseDataType::I32, &testData[0]); - - // Confirm using HDF5IO readDataset that the data is correct - auto readData = hdf5io->readDataset(dataPath); - REQUIRE(readData.shape[0] == 10); - auto readDataTyped = DataBlock::fromGeneric(readData); - REQUIRE(readDataTyped.shape[0] == 10); - REQUIRE(readDataTyped.data == testData); - - // Confirm using lazy read as well - auto readDataWrapper = std::make_unique< - ReadDataWrapper>( - hdf5io, dataPath); - auto readDataGeneric = readDataWrapper->valuesGeneric(); - REQUIRE(readDataGeneric.shape[0] == 10); - auto readDataTypedV2 = readDataWrapper->values(); - REQUIRE(readDataTypedV2.shape[0] == 10); - REQUIRE(readDataTypedV2.data == testData); - - hdf5io->close(); - } -} - TEST_CASE("getH5ObjectType", "[hdf5io]") { // create and open file @@ -1160,4 +1113,215 @@ TEST_CASE("readAttribute", "[hdf5io]") // close file hdf5io.close(); -} \ No newline at end of file +} + +TEST_CASE("HDF5IO; read dataset", "[hdf5io]") +{ + std::vector testData = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + SECTION("read 1D data block of a 1D dataset") + { + // open file + std::string path = getTestFilePath("test_Read1DData1DDataset.h5"); + std::shared_ptr hdf5io = + std::make_shared(path); + hdf5io->open(); + + // Set up test data + std::string dataPath = "/1DData1DDataset"; + SizeType numSamples = 10; + + // Create HDF5RecordingData object and dataset + std::unique_ptr dataset = hdf5io->createArrayDataSet( + BaseDataType::I32, SizeArray {0}, SizeArray {1}, dataPath); + + // Write data block + std::vector dataShape = {numSamples}; + std::vector positionOffset = {0}; + dataset->writeDataBlock( + dataShape, positionOffset, BaseDataType::I32, &testData[0]); + + // Confirm using HDF5IO readDataset that the data is correct + auto readData = hdf5io->readDataset(dataPath); + REQUIRE(readData.shape[0] == 10); + auto readDataTyped = DataBlock::fromGeneric(readData); + REQUIRE(readDataTyped.shape[0] == 10); + REQUIRE(readDataTyped.data == testData); + + // Confirm using lazy read as well + auto readDataWrapper = std::make_unique< + ReadDataWrapper>( + hdf5io, dataPath); + auto readDataGeneric = readDataWrapper->valuesGeneric(); + REQUIRE(readDataGeneric.shape[0] == 10); + auto readDataTypedV2 = readDataWrapper->values(); + REQUIRE(readDataTypedV2.shape[0] == 10); + REQUIRE(readDataTypedV2.data == testData); + + hdf5io->close(); + } + + SECTION("read 2D dataset") + { + // open file + std::string path = getTestFilePath("test_Read2DData2DDataset.h5"); + std::shared_ptr hdf5io = + std::make_shared(path); + hdf5io->open(); + + // Set up test data + std::string dataPath = "/2DData2DDataset"; + SizeType numRows = 2, numCols = 5; + std::vector testData2D = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + + // Create HDF5RecordingData object and dataset + std::unique_ptr dataset = + hdf5io->createArrayDataSet(BaseDataType::I32, + SizeArray {numRows, numCols}, + SizeArray {0, 0}, + dataPath); + + // Write data block + std::vector dataShape = {numRows, numCols}; + std::vector positionOffset = {0, 0}; + dataset->writeDataBlock( + dataShape, positionOffset, BaseDataType::I32, testData2D.data()); + + // Confirm using HDF5IO readDataset that the data is correct + auto readData = hdf5io->readDataset(dataPath); + REQUIRE(readData.shape[0] == numRows); + REQUIRE(readData.shape[1] == numCols); + auto readDataTyped = DataBlock::fromGeneric(readData); + REQUIRE(readDataTyped.shape[0] == numRows); + REQUIRE(readDataTyped.shape[1] == numCols); + REQUIRE(readDataTyped.data == testData2D); + + hdf5io->close(); + } + + SECTION("read dataset with different data types") + { + // open file + std::string path = getTestFilePath("test_ReadDifferentDataTypes.h5"); + std::shared_ptr hdf5io = + std::make_shared(path); + hdf5io->open(); + + // Set up test data for float + std::string floatDataPath = "/FloatDataset"; + std::vector testDataFloat = {1.1f, 2.2f, 3.3f, 4.4f, 5.5f}; + + // Create HDF5RecordingData object and dataset for float + std::unique_ptr floatDataset = + hdf5io->createArrayDataSet( + BaseDataType::F32, SizeArray {5}, SizeArray {0}, floatDataPath); + + // Write float data block + std::vector floatDataShape = {5}; + std::vector floatPositionOffset = {0}; + floatDataset->writeDataBlock(floatDataShape, + floatPositionOffset, + BaseDataType::F32, + testDataFloat.data()); + + // Confirm using HDF5IO readDataset that the float data is correct + auto readFloatData = hdf5io->readDataset(floatDataPath); + REQUIRE(readFloatData.shape[0] == 5); + auto readFloatDataTyped = DataBlock::fromGeneric(readFloatData); + REQUIRE(readFloatDataTyped.shape[0] == 5); + REQUIRE(readFloatDataTyped.data == testDataFloat); + + // Set up test data for double + std::string doubleDataPath = "/DoubleDataset"; + std::vector testDataDouble = {1.1, 2.2, 3.3, 4.4, 5.5}; + + // Create HDF5RecordingData object and dataset for double + std::unique_ptr doubleDataset = + hdf5io->createArrayDataSet( + BaseDataType::F64, SizeArray {5}, SizeArray {0}, doubleDataPath); + + // Write double data block + std::vector doubleDataShape = {5}; + std::vector doublePositionOffset = {0}; + doubleDataset->writeDataBlock(doubleDataShape, + doublePositionOffset, + BaseDataType::F64, + testDataDouble.data()); + + // Confirm using HDF5IO readDataset that the double data is correct + auto readDoubleData = hdf5io->readDataset(doubleDataPath); + REQUIRE(readDoubleData.shape[0] == 5); + auto readDoubleDataTyped = DataBlock::fromGeneric(readDoubleData); + REQUIRE(readDoubleDataTyped.shape[0] == 5); + REQUIRE(readDoubleDataTyped.data == testDataDouble); + + hdf5io->close(); + } + + SECTION("read dataset with an unsupported type") + { + // open file + std::string path = getTestFilePath("test_ReadUnsupportedType.h5"); + std::shared_ptr hdf5io = + std::make_shared(path); + hdf5io->open(); + + // Set up test data for an unsupported type (e.g., string) + std::string dataPath = "/UnsupportedDataset"; + std::vector testDataString = {"a", "b", "c"}; + + // Create HDF5RecordingData object and dataset for string + std::unique_ptr dataset = hdf5io->createArrayDataSet( + BaseDataType::V_STR, SizeArray {3}, SizeArray {0}, dataPath); + + // Attempt to read the dataset and verify that it throws an exception + REQUIRE_THROWS_AS(hdf5io->readDataset(dataPath), std::runtime_error); + + hdf5io->close(); + } +} + +/* +TEST_CASE("HDF5IO; read dataset subset", "[hdf5io]") +{ + SECTION("read dataset with hyperslab selection") + { + // open file + std::string path = getTestFilePath("ReadHyperslabSelection.h5"); + std::shared_ptr hdf5io = + std::make_shared(path); + hdf5io->open(); + + // Set up test data + std::string dataPath = "/2DDataHyperslab"; + SizeType numRows = 3, numCols = 3; + std::vector testData2D = {1, 2, 3, 4, 5, 6, 7, 8, 9}; + + // Create HDF5RecordingData object and dataset + std::unique_ptr dataset = + hdf5io->createArrayDataSet(BaseDataType::I32, + SizeArray {numRows, numCols}, + SizeArray {0, 0}, + dataPath); + + // Write data block + std::vector dataShape = {numRows, numCols}; + std::vector positionOffset = {0, 0}; + dataset->writeDataBlock( + dataShape, positionOffset, BaseDataType::I32, testData2D.data()); + + // Read a hyperslab selection from the dataset + std::vector start = {1, 1}; + std::vector count = {2, 2}; + auto readData = hdf5io->readDataset(dataPath, start, count); + REQUIRE(readData.shape[0] == 2); + REQUIRE(readData.shape[1] == 2); + auto readDataTyped = DataBlock::fromGeneric(readData); + REQUIRE(readDataTyped.shape[0] == 2); + REQUIRE(readDataTyped.shape[1] == 2); + std::vector expectedData = {5, 6, 8, 9}; + REQUIRE(readDataTyped.data == expectedData); + + hdf5io->close(); + } +}*/ \ No newline at end of file From 0765c4ec7e0188b7a48b51b5f91d4c579ff1e535 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 28 Dec 2024 15:23:43 -0800 Subject: [PATCH 18/19] Fix element selection in HDF5IO::readDataset --- src/io/hdf5/HDF5IO.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/io/hdf5/HDF5IO.cpp b/src/io/hdf5/HDF5IO.cpp index 34def54..86bd2bc 100644 --- a/src/io/hdf5/HDF5IO.cpp +++ b/src/io/hdf5/HDF5IO.cpp @@ -408,7 +408,8 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readDataset( { // Check that the dataset exists assert(H5Lexists(m_file->getId(), dataPath.c_str(), H5P_DEFAULT) > 0); - // create the return value to fill + + // Create the return value to fill IO::DataBlockGeneric result; // Create new vectors of type hsize_t for stride and block because @@ -427,23 +428,15 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readDataset( // Get the number of dimensions and their sizes int rank = dataspace.getSimpleExtentNdims(); - // Use a dynamic array as Windows doesn't support variable length arrays std::vector dims(rank); dataspace.getSimpleExtentDims(dims.data(), nullptr); // Store the shape information result.shape.assign(dims.begin(), dims.end()); - // Calculate the total number of elements - size_t numElements = 1; - for (int i = 0; i < rank; ++i) { - numElements *= dims[i]; - } - // Create a memory dataspace for the slice H5::DataSpace memspace; if (!start.empty() && !count.empty()) { - // Use std::vector to create dynamic arrays to ensure Windows built works std::vector offset(rank); std::vector block_count(rank); for (int i = 0; i < rank; ++i) { @@ -459,10 +452,19 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readDataset( block_hsize.empty() ? nullptr : block_hsize.data()); memspace = H5::DataSpace(rank, block_count.data()); + + // Update the shape information based on the hyperslab selection + result.shape.assign(block_count.begin(), block_count.end()); } else { memspace = H5::DataSpace(dataspace); } + // Calculate the total number of elements based on the hyperslab selection + size_t numElements = 1; + for (const auto& c : result.shape) { + numElements *= c; + } + // Read the dataset into a vector of the appropriate type H5::DataType dataType = dataset.getDataType(); if (dataType == H5::PredType::C_S1) { @@ -504,7 +506,8 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readDataset( } else { throw std::runtime_error("Unsupported data type"); } - // return the result + + // Return the result return result; } From 2c8ea8952217a5b2e3267a7d2573e9ae5b6e1a34 Mon Sep 17 00:00:00 2001 From: Oliver Ruebel Date: Sat, 28 Dec 2024 15:24:10 -0800 Subject: [PATCH 19/19] Add unit test for reading hyperslap with HDF5IO::readDataset --- tests/testHDF5IO.cpp | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/testHDF5IO.cpp b/tests/testHDF5IO.cpp index 2fb6272..d0ead51 100644 --- a/tests/testHDF5IO.cpp +++ b/tests/testHDF5IO.cpp @@ -1281,12 +1281,11 @@ TEST_CASE("HDF5IO; read dataset", "[hdf5io]") } } -/* TEST_CASE("HDF5IO; read dataset subset", "[hdf5io]") { SECTION("read dataset with hyperslab selection") { - // open file + // Open file std::string path = getTestFilePath("ReadHyperslabSelection.h5"); std::shared_ptr hdf5io = std::make_shared(path); @@ -1301,27 +1300,39 @@ TEST_CASE("HDF5IO; read dataset subset", "[hdf5io]") std::unique_ptr dataset = hdf5io->createArrayDataSet(BaseDataType::I32, SizeArray {numRows, numCols}, - SizeArray {0, 0}, + SizeArray {numRows, numCols}, dataPath); + REQUIRE(dataset != nullptr); // Write data block std::vector dataShape = {numRows, numCols}; std::vector positionOffset = {0, 0}; - dataset->writeDataBlock( + Status status = dataset->writeDataBlock( dataShape, positionOffset, BaseDataType::I32, testData2D.data()); + REQUIRE(status == Status::Success); // Read a hyperslab selection from the dataset std::vector start = {1, 1}; std::vector count = {2, 2}; + auto readData = hdf5io->readDataset(dataPath, start, count); + + auto readDataTyped = DataBlock::fromGeneric(readData); + std::vector expectedData = {5, 6, 8, 9}; + + // Check the shape of the read data + REQUIRE(readData.shape.size() == 2); REQUIRE(readData.shape[0] == 2); REQUIRE(readData.shape[1] == 2); - auto readDataTyped = DataBlock::fromGeneric(readData); + + // Check the shape of the typed read data + REQUIRE(readDataTyped.shape.size() == 2); REQUIRE(readDataTyped.shape[0] == 2); REQUIRE(readDataTyped.shape[1] == 2); - std::vector expectedData = {5, 6, 8, 9}; + + // Check the data itself REQUIRE(readDataTyped.data == expectedData); hdf5io->close(); } -}*/ \ No newline at end of file +} \ No newline at end of file