Add read for neurodata_types, e.g., Container, TimeSeries

CMakeLists.txt

@@ -25,13 +25,17 @@ add_library(
     src/io/hdf5/HDF5RecordingData.cpp
     src/nwb/NWBFile.cpp
     src/nwb/RecordingContainers.cpp
+    src/nwb/RegisteredType.cpp
     src/nwb/base/TimeSeries.cpp
     src/nwb/device/Device.cpp
     src/nwb/ecephys/ElectricalSeries.cpp
     src/nwb/file/ElectrodeGroup.cpp
     src/nwb/file/ElectrodeTable.cpp
     src/nwb/hdmf/base/Container.cpp
+    src/nwb/hdmf/base/Data.cpp
     src/nwb/hdmf/table/DynamicTable.cpp
+    src/nwb/hdmf/table/VectorData.cpp
+    src/nwb/hdmf/table/ElementIdentifiers.cpp
 )
 
 add_library(aqnwb::aqnwb ALIAS aqnwb_aqnwb)

docs/pages/2_devdocs.dox

@@ -7,6 +7,7 @@
  * - \subpage testing
  * - \subpage dev_docs_page
  * - \subpage nwb_schema_page
+ * - \subpage registered_type_page
  * - \subpage code_of_conduct_page
  * - \subpage license_page
  * - \subpage copyright_page

docs/pages/devdocs/registered_types.dox

@@ -0,0 +1,144 @@
+/**
+ *
+ * \page registered_type_page Implementing a new Neurodata Type
+ *
+ * \tableofcontents
+ *
+ *
+ *
+ * New neurodata_types typically inherit from at least either \ref AQNWB::NWB::Container "Container"
+ * or \ref AQNWB::NWB::Data "Data", or a more specialized type of the two. In any case,
+ * all classes that represent a ``neurodata_type`` defined in the schema should be implemented
+ * as a subtype of \ref AQNWB::NWB::RegisteredType "RegisteredType".
+ *
+ * @section implement_registered_type How to Implement a RegisteredType
+ *
+ * To implement a subclass of \ref AQNWB::NWB::RegisteredType "RegisteredType", follow these steps:
+ *
+ * 1. Include the `RegisteredType.h` header file in your subclass header file.
+ *    @code
+ *    #include "nwb/RegisteredType.hpp"
+ *    @endcode
+ *
+ * 2. Define your subclass by inheriting from \ref AQNWB::NWB::RegisteredType "RegisteredType".
+ *    Ensure that your subclass implements a constructor with the arguments `(const std::string& path, std::shared_ptr<IO::BaseIO> io)`,
+ *    as the "create" method expects this constructor signature.
+ *    @code
+ *    class MySubClass : public AQNWB::NWB::RegisteredType {
+ *    public:
+ *        MySubClass(const std::string& path, std::shared_ptr<IO::BaseIO> io)
+ *            : RegisteredType(path, io) {}
+ *
+ *        // Implement any additional methods or overrides here
+ *    };
+ *    @endcode
+ *
+ * 3. Use the `REGISTER_SUBCLASS` macro to register your subclass. This should usually appear in the header (`hpp`) file as part of the class definition.
+ *    @code
+ *    REGISTER_SUBCLASS(MySubClass, "my-namespace")
+ *    @endcode
+ *
+ * 4. In the corresponding source (`cpp`) file, initialize the static member to trigger the registration.
+ *    @code
+ *    #include "MySubClass.h"
+ *
+ *    // Initialize the static member to trigger registration
+ *    REGISTER_SUBCLASS_IMPL(MySubClass)
+ *    @endcode
+ *
+ * \warning
+ * To ensure proper function on read, the name of the class should match the name of the
+ * ``neurodata_type`` as defined in the schema. Similarly, "my-namespace" should match
+ * the name of the namespace in the schema (e.g., "core", "hdmf-common"). In this way
+ * we can look up the corresponding class for an object in a file based on the
+ * ``neurodata_type`` and ``namespace`` attributes stored in the file.
+ *
+ *
+ * **Example:**
+ *
+ * *MySubClass.h*
+ * @code
+ * #pragma once
+ * #include "RegisteredType.h"
+ *
+ * class MySubClass : public AQNWB::NWB::RegisteredType {
+ * public:
+ *     MySubClass(const std::string& path, std::shared_ptr<IO::BaseIO> io)
+ *         : RegisteredType(path, io) {}
+ *
+ *     void doSomething() const override {
+ *         std::cout << "MySubClass doing something!" << std::endl;
+ *     }
+ *
+ *     REGISTER_SUBCLASS(MySubClass, "my-namespace")
+ * };
+ * @endcode
+ *
+ * *MySubClass.cpp*
+ * @code
+ * #include "MySubClass.h"
+ *
+ * // Initialize the static member to trigger registration
+ * REGISTER_SUBCLASS_IMPL(MySubClass)
+ * @endcode
+ *
+ * @section type_registry How the Type Registry in RegisteredType Works
+ *
+ * The type registry in \ref AQNWB::NWB::RegisteredType "RegisteredType" allows for dynamic creation of registered subclasses by name. Here is how it works:
+ *
+ * 1. **Registry Storage**:
+ *    - The registry is implemented using 1) an `std::unordered_set` to store subclass names (which can be
+ *      accessed via \ref AQNWB::NWB::RegisteredType::getRegistry "getRegistry()") and
+ *      2) an `std::unordered_map` to store factory functions for creating instances of the subclasses
+ *      (which can be accessed via \ref AQNWB::NWB::RegisteredType::getFactoryMap() "getFactoryMap()").
+ *      The factory methods are the required constructor that uses the io and path as input.
+ *    - These are defined as static members within the \ref AQNWB::NWB::RegisteredType "RegisteredType" class.
+ *
+ * 2. **Registration**:
+ *    - The \ref AQNWB::NWB::RegisteredType::registerSubclass "registerSubclass" method is used to add a
+ *      subclass name and its corresponding factory function to the registry.
+ *    - This method is called via the `REGISTER_SUBCLASS` macro, which defines a static method (`registerSubclass()`)
+ *      and static member (`registered_`) to trigger the registration when the subclass is loaded.
+ *
+ * 3. **Dynamic Creation**:
+ *    - The \ref AQNWB::NWB::RegisteredType::create "create" method is used to create an instance of a registered subclass by name.
+ *    - This method looks up the subclass name in the registry and calls the corresponding factory function to create an instance.
+ *
+ * 4. **Automatic Registration**:
+ *    - The `REGISTER_SUBCLASS_IMPL` macro initializes the static member (`registered_`), which triggers the
+ *      \ref AQNWB::NWB::RegisteredType::registerSubclass "registerSubclass" method
+ *      and ensures that the subclass is registered when the program starts.
+ *
+ * 5. **Class Name and Namespace Retrieval**:
+ *    - The \ref AQNWB::NWB::RegisteredType::getTypeName "getTypeName" and
+ *      \ref AQNWB::NWB::RegisteredType::getNamespace "getNamespace" return the string name the
+ *      class and namespace, respectively. The `REGISTER_SUBCLASS` macro implements and automatic
+ *      override of the methods to ensure the appropriate type and namespace string are returned.
+ *      These methods should, hence, not be manually overridden by subclasses, to ensure consistency
+ *      in type identification.
+ *
+ *
+ * @section use_registered_type_registry How to Use the RegisteredType Registry
+ *
+ * The \ref AQNWB::NWB::RegisteredType "RegisteredType" registry allows for dynamic creation and management of registered subclasses. Here is how you can use it:
+ *
+ * 1. **Creating Instances Dynamically**:
+ *    - Use the \ref AQNWB::NWB::RegisteredType::create "create" method to create an instance of a registered subclass by name.
+ *    - This method takes the subclass name, path, and a shared pointer to the IO object as arguments. This
+ *      illustrates how we can read a specific typed object in an NWB file.
+ *    \snippet tests/examples/test_RegisteredType_example.cpp example_RegisterType_get_type_instance
+ *
+ * 2. **Retrieving Registered Subclass Names**:
+ *    - Use the \ref AQNWB::NWB::RegisteredType::getRegistry "getRegistry" method to retrieve the
+ *      set of registered subclass names.
+ *      \snippet tests/examples/test_RegisteredType_example.cpp example_RegisterType_get_registered_names
+ *
+ * 3. **Retrieving the Factory Map**:
+ *    - Use the \ref AQNWB::NWB::RegisteredType::getFactoryMap "getFactoryMap" method to retrieve
+ *      the map of factory functions for creating instances of registered subclasses.
+ *      \snippet tests/examples/test_RegisteredType_example.cpp example_RegisterType_get_registered_factories
+ *
+ * **Full Example**
+ *
+ *  \snippet tests/examples/test_RegisteredType_example.cpp example_RegisterType_full
+ */

docs/pages/userdocs/read.dox

@@ -233,7 +233,7 @@
  *
  * \snippet tests/examples/test_ecephys_data_read.cpp example_read_create_file_snippet
  *
- * \subsection read_design_example_read_during Read data during recording
+ * \subsection read_design_example_read_during Reading Datasets and Attributes
  *
  * \paragraph read_design_example_laxy_read Lazy data access
  *
@@ -249,13 +249,13 @@
  *
  * \paragraph read_design_example_load_data Read data into memory
  *
- * To access the data values of a data, we can then use the \ref AQNWB::IO::ReadDataWrapper::valuesGeneric "valuesGeneric"
+ * To load the data values, we can then use the \ref AQNWB::IO::ReadDataWrapper::valuesGeneric "valuesGeneric"
  * and \ref AQNWB::IO::ReadDataWrapper::values "values" methods, which load the data as generic (untyped) or typed
  * data, respectively.
  *
  * \snippet tests/examples/test_ecephys_data_read.cpp example_read_get_datablock_snippet
  *
- * The data is here represented as a \ref AQNWB::IO::DataBlock "DataBlock", which stores the data as 1-dimensionsal
+ * The data is here represented as a \ref AQNWB::IO::DataBlock "DataBlock", which stores the data as a 1-dimensionsal
  * vector along with the shape of the data. E.g, here we validate the data against the original mock data:
  *
  * \snippet tests/examples/test_ecephys_data_read.cpp example_read_validate_datablock_snippet
@@ -309,21 +309,20 @@
  * still read the data via the \ref AQNWB::IO::ReadDataWrapper::valuesGeneric "valuesGeneric"
  * to load the data first in untyped form. When loading the data, the I/O backend determines
  * the data type and allocates memory appropriately. The actual data type is then stored
- * in \ref AQNWB::IO::DataBlockGeneric::typeIndex "typeIndex" variable of our data block.
+ * in the \ref AQNWB::IO::DataBlockGeneric::typeIndex "typeIndex" variable of our data block.
  * We can then convert our \ref AQNWB::IO::DataBlockGeneric "DataBlockGeneric" to a
  * \ref AQNWB::IO::DataBlock "DataBlock<DTYPE>" with a specific data type via
- * \ref AQNWB::IO::DataBlock::fromGeneric "DataBlock.fromGeneric()".
+ * \ref AQNWB::IO::DataBlock::fromGeneric "DataBlock<dtype>::fromGeneric()".
  *
  * \snippet tests/examples/test_ecephys_data_read.cpp example_read_get_data_wrapper_as_generic_snippet
  *
  * \note
  * In most cases, we should not need runtime checking of types in the context of
- * specific data acquisition systems. This is likely mostly relevant if one wants to consume
+ * specific data acquisition systems. This is mostly likely relevant if one wants to consume
  * arbitrary NWB files that may use different data types. One approach to implement
  * behavior for types determined at runtime is to define a mapping of the type
  * information to the corresponding statically type functionality, e.g., via
  * ``switch/case`` logic or by using a map for lookup, such as:
- *
  * \code{.cpp}
  * DataBlockGeneric dataValuesGeneric = readDataWrapperGeneric->valuesGeneric();
  * // Map to associate std::type_index with corresponding type-specific functions
@@ -335,26 +334,46 @@
  * // Use the map to process the data with the approbriate type
  * auto it = typeMap.find(dataValuesGeneric.typeIndex);
  * if (it != typeMap.end()) {
- *       it->second(dataValuesGeneric); // call the correct proceesData function
+ *       it->second(dataValuesGeneric); // call the correct processData function
  * } else {
  *       std::cout << "Unsupported type" << std::endl;
  * }
  * \endcode
  *
+ *
+ * \subsection read_design_example_read_posthoc Reading a Registered Type (e.g., a TimeSeries)
+ *
+ * To read from a ``neurodata_type`` object from and existing file we can use the
+ * \ref AQNWB::NWB::RegisteredType::create "RegisteredType::create"  factory methods
+ * to conveniently construct an instance of the corresponding class in AqNWB.
+ *
+ *  \snippet tests/examples/test_ecephys_data_read.cpp  example_read_only_snippet
+ *
+ * \note
+ * \ref AQNWB::NWB::RegisteredType::create "RegisteredType::create" comes in a few
+ * different flavours:
+ * 1. When passing only 1) ``path`` and 2) ``io`` (as in the example above), AqNWB
+ *    reads the ``neurodata_type`` and ``namespace`` attributes from the NWB file to
+ *    automatically determine the class to use to represent the type.
+ * 2. When passing the 1) ``fullname`` (e.g., ``core::ElectricalSeries``),  1) ``path`` and 2) ``io``
+ *    AqNWB looks up the class to use in \ref AQNWB::NWB::RegisteredType "RegisteredType's" type
+ *    registry (see also \ref use_registered_type_registry )
+ * 3. When passing the class to use as template parameter, e.g.,
+ *    ``create<AQNWB::NWB::ElectricalSeries>(path, io);`` the
+ *    instance is being constructed using the common constructor, i.e., this
+ *    is equivalent to creating the object via ``ElectricalSeries(path, io)``
+ * Option 1 and 2 instantiates the specific type (e.g., ``ElectricalSeries``) but return
+ * a generic \ref AQNWB::NWB::RegisteredType "RegisteredType" pointer that we can
+ * cast to the specific type if necessary. Option 3 creates and returns a pointer to
+ * the specific type directly.
+ *
  * \subsection read_design_example_stop_recording Finalize the recording
  *
  * Next we stop the recording and close the file so we can show how we can read from the file
  * we just created.
  *
  * \snippet tests/examples/test_ecephys_data_read.cpp example_read_finish_recording_snippet
  *
- * \subsection read_design_example_read_posthoc Read data from an existing file
- *
- * To read from an existing file we simply need to create the I/O object and the construct the
- * Container object we want to read.
- *
- *  \snippet tests/examples/test_ecephys_data_read.cpp  example_read_only_snippet
- *
  */
 
 

src/io/hdf5/HDF5IO.cpp

@@ -212,8 +212,8 @@ std::vector<std::string> HDF5IO::readStringDataHelper(
 
 AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute(const std::string& dataPath)
 {
-  // create the return value to fill
-  IO::DataBlockGeneric result;
+  // Create the return value to fill
+  AQNWB::IO::DataBlockGeneric result;
   // Read the attribute
   auto attributePtr = this->getAttribute(dataPath);
   // Make sure dataPath points to an attribute
@@ -223,7 +223,7 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute(const std::string& dataPath)
 
   // Determine the shape of the attribute
   H5::DataSpace dataspace = attribute.getSpace();
-  SizeType rank = dataspace.getSimpleExtentNdims();
+  int rank = dataspace.getSimpleExtentNdims();
   if (rank == 0) {
     // Scalar attribute
     result.shape.clear();
@@ -238,21 +238,40 @@ AQNWB::IO::DataBlockGeneric HDF5IO::readAttribute(const std::string& dataPath)
   }
 
   // Determine the size of the attribute from the shape
-  // For some reason using "size_t numElements = attribute.getStorageSize();"
-  // seems to not always give the expected size (but may be larger), so
-  // we calculate the number of elements from the shape instead
   size_t numElements = 1;  // Scalar (default)
-  if (result.shape.size() > 0)  // N-dimensional array
-  {
+  if (result.shape.size() > 0) {  // N-dimensional array
     for (const auto v : result.shape) {
       numElements *= v;
     }
   }
 
   // Read the attribute into a vector of the appropriate type
-  if (dataType == H5::PredType::C_S1) {
-    result.data = readStringDataHelper(attribute, numElements);
-    result.typeIndex = typeid(std::string);
+  if (dataType.getClass() == H5T_STRING) {
+    if (dataType.isVariableStr()) {
+      // Handle variable-length strings
+      std::vector<std::string> stringData;
+      try {
+        // Allocate a buffer to hold the variable-length string data
+        char* buffer;
+        attribute.read(dataType, &buffer);
+
+        // Convert the buffer to std::string
+        stringData.emplace_back(buffer);
+
+        // Free the memory allocated by HDF5
+        H5Dvlen_reclaim(
+            dataType.getId(), dataspace.getId(), H5P_DEFAULT, &buffer);
+      } catch (const H5::Exception& e) {
+        throw std::runtime_error(
+            "Failed to read variable-length string attribute: "
+            + std::string(e.getDetailMsg()));
+      }
+      result.data = stringData;
+      result.typeIndex = typeid(std::string);
+    } else {
+      result.data = readStringDataHelper(attribute, numElements);
+      result.typeIndex = typeid(std::string);
+    }
   } else if (dataType == H5::PredType::NATIVE_DOUBLE) {
     result.data = readDataHelper<double>(attribute, numElements);
     result.typeIndex = typeid(double);

src/nwb/NWBFile.cpp

@@ -26,12 +26,21 @@ constexpr SizeType CHUNK_XSIZE = 2048;
 std::vector<SizeType> NWBFile::emptyContainerIndexes = {};
 
 // NWBFile
+// Initialize the static registered_ member to trigger registration
+REGISTER_SUBCLASS_IMPL(NWBFile)
 
 NWBFile::NWBFile(std::shared_ptr<IO::BaseIO> io)
     : Container("/", io)
 {
 }
 
+NWBFile::NWBFile(const std::string& path, std::shared_ptr<IO::BaseIO> io)
+    : Container("/", io)  // Always use "/" for the path
+{
+  std::cerr << "NWBFile object is always the root. Path must be /" << std::endl;
+  assert(path == "/");
+}
+
 NWBFile::~NWBFile() {}
 
 Status NWBFile::initialize(const std::string& identifierText,
@@ -59,7 +68,8 @@ Status NWBFile::createFileStructure(const std::string& identifierText,
     return Status::Failure;
   }
 
-  io->createCommonNWBAttributes("/", "core", "NWBFile", "");
+  io->createCommonNWBAttributes(
+      this->path, this->getNamespace(), this->getTypeName(), "");
   io->createAttribute(AQNWB::SPEC::CORE::version, "/", "nwb_version");
 
   io->createGroup("/acquisition");

src/nwb/NWBFile.hpp

@@ -26,12 +26,20 @@ namespace AQNWB::NWB
 class NWBFile : public Container
 {
 public:
+  // Register the ElectrodeTable as a subclass of Container
+  REGISTER_SUBCLASS(NWBFile, "core")
+
   /**
    * @brief Constructor for NWBFile class.
    * @param io The shared pointer to the IO object.
    */
   NWBFile(std::shared_ptr<IO::BaseIO> io);
 
+  /** @brief Required constructor so we can call RegisteredType::create but the
+   * path cannot be set
+   */
+  NWBFile(const std::string& path, std::shared_ptr<IO::BaseIO> io);
+
   /**
    * @brief Deleted copy constructor to prevent construction-copying.
    */